{ "$schema": "./torchwright_schematic_v1.schema.json", "artifact": { "architecture": "phi3", "files": { "config.json": { "bytes": 999, "sha256": "50b1da6f3ed4ac332fa6210f6fd634022f8e3a75592b0ea392d2d8f994b167b7" }, "generation_config.json": { "bytes": 105, "sha256": "6ebdfc7ea4a3b172e5baf014667c957a7eb90ea465d05178b801cc2ab7b6bdc4" }, "model-00001-of-00028.safetensors": { "bytes": 151036360, "sha256": "824b74c24d3e2009bf1746b4960fcd4fe27c2adb9aae1afc04173f5776de3210" }, "model-00002-of-00028.safetensors": { "bytes": 151036360, "sha256": "c1377379e03d5d0dffa320408c70f6ff508827d1ac8a5e7664bcf1771a95b573" }, "model-00003-of-00028.safetensors": { "bytes": 151036360, "sha256": "86c478de5832354dd97a0b3199e70b00894b985076a625ab32a3d147f4ced288" }, "model-00004-of-00028.safetensors": { "bytes": 151036360, "sha256": "b37dd309e6bf011f920ff002f600043ce018d8f83347a9a943503445902eb8ff" }, "model-00005-of-00028.safetensors": { "bytes": 151036360, "sha256": "341f54300d6928cdcdb6346deb4efc9ef3250950d5c3a4df9cf53fc2dbf65d81" }, "model-00006-of-00028.safetensors": { "bytes": 151036360, "sha256": "58d744c6e59ae00b335045442594ab07568150a3ba42bb449a3644e04b19d471" }, "model-00007-of-00028.safetensors": { "bytes": 151036360, "sha256": "8ebb2b437c55a36a23b628682cb14394d7cf7bc0d4bf7a22474ada367c20ac1a" }, "model-00008-of-00028.safetensors": { "bytes": 151036360, "sha256": "25eb7aca11722819ef664fbfcf1258b2af29d5cbc6e9b27ee98d7a5052c38c4d" }, "model-00009-of-00028.safetensors": { "bytes": 151036360, "sha256": "1d872c7361d628b8ca0164ba398eb83e142837fae34b23b4c3a5aeb0c5eebba0" }, "model-00010-of-00028.safetensors": { "bytes": 151036360, "sha256": "75130f96bdb2119393c6d5de6a936be72e3eb619f8a8f662f164d9857c4661ba" }, "model-00011-of-00028.safetensors": { "bytes": 151036360, "sha256": "b22f70402c83c6192425ee7aae158b70207d049b0e59a00d9f2c169000110710" }, "model-00012-of-00028.safetensors": { "bytes": 151036360, "sha256": "69b0ab8122e3c304f71276cec78469b71c31bc73bbdbbe73c1f5d2ec89509ace" }, "model-00013-of-00028.safetensors": { "bytes": 151036360, "sha256": "c90f23ad6f7cd30d4a59a568a840bac394e660a882a4c9460d1ef36c2bc749af" }, "model-00014-of-00028.safetensors": { "bytes": 151036360, "sha256": "c5a0b3d0572e02d147f25fc6e0e5f7106ef727e12af01592e45d467eeed4ef22" }, "model-00015-of-00028.safetensors": { "bytes": 151036360, "sha256": "1d469d1857a90fc09a347dfd4857e8a9de3ca959ac81e14c61e768431e2f0888" }, "model-00016-of-00028.safetensors": { "bytes": 151036360, "sha256": "f56a50558b2b7f67e4d4b7515e3876186d03fd40a46caecf8e92744ae43ae269" }, "model-00017-of-00028.safetensors": { "bytes": 151036360, "sha256": "60c66b57f8ab4275a357c615f414c6c52a9c3dce4d0829f2db96d857c7b6fe21" }, "model-00018-of-00028.safetensors": { "bytes": 151036360, "sha256": "f2a43d94976176b122b783a9d135e200b510c26ad62602499ba6d8b18e44a86d" }, "model-00019-of-00028.safetensors": { "bytes": 151036360, "sha256": "cb61c0f0ed417db14506ff3e31b697d7928f22edf8397391991938ec452c328c" }, "model-00020-of-00028.safetensors": { "bytes": 151036360, "sha256": "bdbd23c8004d6cb0e3b6e55d58496993254c6a2e73b79709f8d50d019691ed23" }, "model-00021-of-00028.safetensors": { "bytes": 151036360, "sha256": "3fc1a31b0b50956efc0b49762e036bbe16f8567097651e15cf6fef4cdf193341" }, "model-00022-of-00028.safetensors": { "bytes": 151036360, "sha256": "6925ac3c1547dba03c096dbffee4fcb67f288f0afe306fdd95a1a369a01af17f" }, "model-00023-of-00028.safetensors": { "bytes": 151036360, "sha256": "0475c2f01c6ac7a747f543600d0b975f70d0d75de2d9a3c22daed96cf728b13b" }, "model-00024-of-00028.safetensors": { "bytes": 151036360, "sha256": "fc573f07e3980e2d2d20785b09c6c052ba2fb31a6fab7ba3ebce99bdc258b284" }, "model-00025-of-00028.safetensors": { "bytes": 151036360, "sha256": "2897412382107263d5a40c693fdb4949945f241917a6abe59872fb036a494216" }, "model-00026-of-00028.safetensors": { "bytes": 151036360, "sha256": "0c1bc6338ba053363703b6300518a62a4c983189af0440cc9e1d4bd8e4f187be" }, "model-00027-of-00028.safetensors": { "bytes": 151036360, "sha256": "7927da63e75089667920673db8fb4c2542617e1dbe2515bbc9ea2af2c4008730" }, "model-00028-of-00028.safetensors": { "bytes": 603984, "sha256": "edb3de6239bde555190ac6dabe7d2bbe171a6fd20f88e78c83693948d8b77f73" }, "model.safetensors.index.json": { "bytes": 13078, "sha256": "a315bc8739bc2c92885fbd03a8235526d73f885cd80d37bbf688914d51c9b1f9" }, "tokenizer.json": { "bytes": 2041, "sha256": "a63388c8362fc109382b66506b666d798b7e0170828d0047d16790978a14b5f0" }, "tokenizer_config.json": { "bytes": 199, "sha256": "6f9df026786c39bfda75becfa45ea655ccad3ee7bf4c4e186777e11aa930758b" }, "torchwright_schematic_support.npz": { "bytes": 818744, "sha256": "ba74247d8781a6aa68ee715776792e9a4ab703b74732941d3a81cb4147735615" }, "torchwright_schematic_v1.schema.json": { "bytes": 3678, "sha256": "1881c7400c00ee66e6e911c7416ceb573b31129cf24bb80d4055b9e2c647ec7f" } }, "kind": "huggingface_causal_lm_bundle" }, "build": { "compiler_code_sha256": "639e918d8b7011d6f247de3e5696cc4faaa3b154fe081a266f956ddaa9e68e1c", "options": { "activation": "swish", "bias": false, "collapse_piecewise_linear": true, "collapse_univariate": true, "d_head": 64, "d_hidden_capacity": 4096, "d_model": 2048, "n_heads_capacity": 32, "optimize": 2, "trim_heads": true }, "schedule_fingerprint": "36f8c8170aae72b9d0f1d7be0dbc26066a994405320cf34571b7158624d7eed0", "schedule_provenance": { "delivery": "fresh", "optimize": 2, "selected_is_optimal": true, "selected_objective": 27, "selected_objective_blocks": [ 27, 0 ], "selected_origin": "solver", "solver_best_bound": 27.0, "solver_is_optimal": true, "solver_objective": 27, "solver_status": "OPTIMAL" }, "torchwright_version": "unknown" }, "format": "torchwright.schematic.v1", "graphs": { "internal_nodes": [ { "id": "i:0", "name": "rope_self_match_const_one", "op": "LiteralValue", "purpose": "compiler_internal", "width": 1 } ], "lowered": { "nodes": [ { "annotation": null, "id": "l:0", "inputs": [ "l:1" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:0" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=146.70925247975623))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:1", "inputs": [ "l:2", "l:10", "l:451" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:2", "inputs": [ "l:3", "l:6", "l:7" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:2" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:3", "inputs": [ "l:4", "l:5" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:3" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:4", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:4" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:5", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:5" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:6", "inputs": [ "l:7", "l:9" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:6" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:7", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3eb8480c9ef49139f499acac7ad953fae7d8c289cb60b001375681e6691ca7d1", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:7" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:8", "inputs": [], "name": null, "op": "Embedding", "parameters": [ { "dtype": "float32", "name": "table", "sha256": "6bbd3059883cf8b4699c1da2e25ef31211d490f885fcaac1f73475d14b493c32", "shape": [ 18, 17 ] } ], "scheduling_predecessors": [], "semantics": { "embedding_width": 17, "input_name": "embedding_input", "vocab_size": 18 }, "source_nodes": [ "s:8" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:9", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:9" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:10", "inputs": [ "l:11" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d1eb096a138388c9ee051d92d5ac6d8e55f4754d7e7650310fcab089dad79925", "shape": [ 136, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:10" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:11", "inputs": [ "l:12", "l:362", "l:383", "l:401", "l:411", "l:421", "l:431", "l:441" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:11" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.67731311993905, "upper_unbounded": false } }, "width": 136 }, { "annotation": null, "id": "l:12", "inputs": [ "l:13" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:12" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=18.247419462656246))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:13", "inputs": [ "l:14", "l:23" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:13" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:14", "inputs": [ "l:15" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:14" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:15", "inputs": [ "l:16", "l:22" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:16" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:16", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:17" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.9956489921875, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:17", "inputs": [ "l:18" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:18" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3418431447269631, "lower_unbounded": false, "upper": 9.50850981139363, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:18", "inputs": [ "l:19" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6c50dd4f13b1539d4f43be532766dbdf7fb95eaa3941166aa039680e2e300a9d", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6b66e1d03190b21246c3ec82a52fd11b0466bbcf48208c4e3f55fb2eaec6cc20", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "483afcc6ea8b039d16cfd49cb47e300e3ec28b944e85daf9d068c4c79a838894", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "60ace6d30c5997f9dc18846fc3aac46f4e0ed2220c14b4c15fcb4f3b0cb7a036", "shape": [ 32, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "source_nodes": [ "s:19" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6581568552730369, hi=10.50850981139363))" } ], "claimed_range": { "lower": 0.6581568552730369, "lower_unbounded": false, "upper": 10.50850981139363, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6581568552730369, "lower_unbounded": false, "upper": 10.50850981139363, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:19", "inputs": [ "l:20", "l:2", "l:21" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:20" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:20", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:21" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:21", "inputs": [ "l:7" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:22" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:22", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:24" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:23", "inputs": [ "l:24" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:26" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:24", "inputs": [ "l:25" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:27" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:25", "inputs": [ "l:26", "l:348" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:28" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:26", "inputs": [ "l:27" ], "name": "collapse_pl_Add#399_in_range_0_46", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "7580ae8af901cf34a2b10c1f27c0982c8b3075a6771a529f3d120d819901dee9", "shape": [ 46 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "cd78aa3e39d231259d577b69360aab9be0901ee17b98b174e5082aa1fa40e30d", "shape": [ 46, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e2c1e476e4d39f998a1474e3e82df6cd962dac65d311e65d5a0c29d53b6dee2c", "shape": [ 12 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "7bfd681b2968770c559fa175fc27c4faab6c6218fef01a3adf715374899523f6", "shape": [ 46, 12 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 46 }, "source_nodes": [ "s:29" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005074638340876, hi=1.0005073449969))" } ], "claimed_range": { "lower": -1.0005074638340876, "lower_unbounded": false, "upper": 1.0005073449969, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005074638340876, "lower_unbounded": false, "upper": 1.0005073449969, "upper_unbounded": false } }, "width": 12 }, { "annotation": null, "id": "l:27", "inputs": [ "l:28", "l:344" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:31" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.023930542968749968, "lower_unbounded": false, "upper": 11.023930542968749, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:28", "inputs": [ "l:29" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "072e3304b03423a4767d28c5fed09f81d5190ff60a3d078c6c1350eeb8bee28b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "76cb54bedbda438629b40aa213e8fcdaa3a961e6c305ff2df2b28ab97d143335", "shape": [ 5, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:32" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.010877519531249824, "lower_unbounded": false, "upper": 5.01087751953125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:29", "inputs": [ "l:30", "l:243", "l:257", "l:280", "l:329" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "l:30", "inputs": [ "l:31" ], "name": "collapse_pl_equals_vector_compare_0_4", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0716ec39c7103bc68885c4c38eab888048f12e4a511d3c97f652324e25e1a490", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a0c210f3fab053052f1b881e15ec97fe9fb4c75e26c09087eb9a2121b689d359", "shape": [ 4, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1893dbc217bd565883e63e6a5b175b772bbfd7ebd0b9c0394e0663245cca60b", "shape": [ 4, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 4 }, "source_nodes": [ "s:36" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=1.0005))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:31", "inputs": [ "l:32" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:41" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:32", "inputs": [ "l:33" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:42" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:33", "inputs": [ "l:34", "l:60", "l:69" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:43" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:34", "inputs": [ "l:35" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:44" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:35", "inputs": [ "l:36", "l:59" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:45" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:36", "inputs": [ "l:37", "l:40", "l:46" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:46" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:37", "inputs": [ "l:38", "l:39" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:47" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:38", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:48" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:39", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:49" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:40", "inputs": [ "l:41", "l:58" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:50" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:41", "inputs": [ "l:42" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:51" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:42", "inputs": [ "l:43", "l:51" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:53" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:43", "inputs": [ "l:44" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9fe7252ae4cff6be4d42fdf46fd404e91cf1503a785d6ee2cfc0779c6e4c7a57", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:54" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:44", "inputs": [ "l:45", "l:47", "l:49" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:56" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:45", "inputs": [ "l:46" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:57" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:46", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ef7b1f9e5a2d907c7ff2496a32dc2d4879721c404cc224a31671940ba846c4e2", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:58" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:47", "inputs": [ "l:48" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:59" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:48", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "82c5b2a5f705dd388e12fac162246ad059cfa99089bdd1df5103928849793813", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:60" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:49", "inputs": [ "l:50" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:61" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:50", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2e5d5c3fc5f63a23d29988ab3982c8da9a63a8cd2639d8ca27ab5390529327d9", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:62" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:51", "inputs": [ "l:52" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:63" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:52", "inputs": [ "l:53", "l:56", "l:7" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:64" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:53", "inputs": [ "l:54", "l:55" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:65" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:54", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:66" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:55", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:67" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:56", "inputs": [ "l:7", "l:57" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:68" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:57", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:69" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:58", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:70" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:59", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:71" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:60", "inputs": [ "l:61" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:72" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:61", "inputs": [ "l:62", "l:68" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:73" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:62", "inputs": [ "l:63", "l:66", "l:48" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:74" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:63", "inputs": [ "l:64", "l:65" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:75" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:64", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:76" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:65", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:77" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:66", "inputs": [ "l:41", "l:67" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:78" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:67", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:79" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:68", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:80" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:69", "inputs": [ "l:70" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:81" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:70", "inputs": [ "l:71", "l:77" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:82" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:71", "inputs": [ "l:72", "l:75", "l:50" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:83" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:72", "inputs": [ "l:73", "l:74" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:84" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:73", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:85" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:74", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:86" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:75", "inputs": [ "l:41", "l:76" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:87" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:76", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:88" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:77", "inputs": [ "l:78" ], "name": "collapse_pl_Add#371_fused_fused_add_const_multiply_const_onehot_lookup_select_0_62", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "622a859af160510b88c05fe2f1fa65a6d349ce51a925b2a4c1acd8b654c98ac1", "shape": [ 62 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "836a6462892e412c7f6dc69bccb05f7e3a6928f4251baf32a48eef5837a2b2b2", "shape": [ 62, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "d924a8202c635e764e88f6f43f3917d3af0f55055f469676be5dcc2f568e234d", "shape": [ 62, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 62 }, "source_nodes": [ "s:89" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005074638340877642, hi=1.000507332675651))" } ], "claimed_range": { "lower": -0.0005074638340877642, "lower_unbounded": false, "upper": 1.000507332675651, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005074638340877642, "lower_unbounded": false, "upper": 1.000507332675651, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:78", "inputs": [ "l:79", "l:178" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:94" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 15.002175503906251, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:79", "inputs": [ "l:80" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:95" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:80", "inputs": [ "l:81" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:98" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:81", "inputs": [ "l:82", "l:134" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:99" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:82", "inputs": [ "l:83", "l:86", "l:88" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:100" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:83", "inputs": [ "l:84", "l:85" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:101" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:84", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:102" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:85", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:103" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:86", "inputs": [ "l:7", "l:87" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:104" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:87", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:105" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:88", "inputs": [ "l:89", "l:108", "l:131" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:106" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:89", "inputs": [ "l:90" ], "name": "collapse_pl_Attn#152_m224_0_9", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "3fdefc9ed00f00dea0599794d1c9a69cee3f0f6849c87508fff2da9d6bbb3ddb", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b05e3b5ab8b2d409e8b3d5f16ea3c8191e3638bbb3168f525a428a2af82fba6b", "shape": [ 9, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "eaa82dd1f86788890259eaf455712f67cccbe5433ad98eac3cad71e322d8d347", "shape": [ 9, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:107" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009574060440063))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:90", "inputs": [ "l:91", "l:94", "l:96" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:114" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:91", "inputs": [ "l:92", "l:93" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:115" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:92", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:116" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:93", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:117" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:94", "inputs": [ "l:41", "l:95" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:118" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:95", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:119" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:96", "inputs": [ "l:97" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:120" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:97", "inputs": [ "l:98" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "86ace85102b9d8a1c0886e9cf2cbb181c89f8ca60093d8ccbe6cbd12a754ad1b", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5870a33a3d94db2db2aaa92ac9edb125e7803a503405c199042a2f0c6d461a6b", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "41d043a7c0f00b8b39f1365727f09ede61e4b865694b6dc1d4acbbd97029d11b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a4d06da2357859d57b942afb3157f9957beb9485b57a8ae112cd6e8f5e6b3764", "shape": [ 32, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "source_nodes": [ "s:121" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6633250767579517, hi=5.503341589908715))" } ], "claimed_range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:98", "inputs": [ "l:99", "l:100", "l:107" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:122" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:99", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:123" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:100", "inputs": [ "l:101", "l:104", "l:105" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:124" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:101", "inputs": [ "l:102", "l:103" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:125" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:102", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:126" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:103", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:127" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:104", "inputs": [ "l:105", "l:106" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:128" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:105", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "e79f394c9a536f65c56b29d1e593771c6e332b653623054cfde6b4c6056348bf", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:129" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:106", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:130" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:107", "inputs": [ "l:105" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:131" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:108", "inputs": [ "l:109", "l:128" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:136" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.5118727628515622, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:109", "inputs": [ "l:110", "l:127" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:137" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.0093727628515623, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:110", "inputs": [ "l:111" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "acc2e4c62e9b92c67550621c1d3c42498eab8b87200286819aae131530c4dc41", "shape": [ 3, 4 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e5cd6f63a143d03bf475cd19b883bbdecec4efb62d8303f209b0dde6ae0565cd", "shape": [ 3, 3 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "ba88de7e97f50c001a9d0c4e5c690cd5e4d90c7e5a7b64b96b485a3847228885", "shape": [ 3, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 3 }, "source_nodes": [ "s:138" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.004351007812499974, hi=1.0043510078125))" } ], "claimed_range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:111", "inputs": [ "l:112", "l:122" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:139" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:112", "inputs": [ "l:113" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:140" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:113", "inputs": [ "l:114", "l:115" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:142" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:114", "inputs": [ "l:8" ], "name": "map_to_table", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "2c34ce1df23b838c5abf2a7f6437cca3d3067ed509ff25f11df6b11b582b51eb", "shape": [ 10 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "eee183eff6e72006bb7ace8c5ae3f42029abaa0004524c553789b6cca053a22c", "shape": [ 10, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e7a2f4b7a886aabf181b94c496dabd00bd3f8da32a01f3da03e6d6868564aed6", "shape": [ 10, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 10 }, "source_nodes": [ "s:143" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-21.0, hi=19.0))" } ], "claimed_range": { "lower": -21.0, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:115", "inputs": [ "l:116" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:144" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:116", "inputs": [ "l:117", "l:120", "l:41" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:145" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:117", "inputs": [ "l:118", "l:119" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:146" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:118", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:147" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:119", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:148" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:120", "inputs": [ "l:41", "l:121" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:149" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:121", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:150" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:122", "inputs": [ "l:123" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "fbfe4ca61ec299680879894a16ca776f81f23454312659ac8736249aa1e9722b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:151" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:123", "inputs": [ "l:124" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "source_nodes": [ "s:153" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:124", "inputs": [ "l:125", "l:126" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:154" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:125", "inputs": [ "l:96" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:155" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:126", "inputs": [ "l:125" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:156" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:127", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:157" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:128", "inputs": [ "l:129" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6ad01752d1a3928862bafa2e6bbe6b2df45d614b086e065fdacf50cb7180f6f3", "shape": [ 4, 5 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c91315e209cc3cd1c12fae1264b1597c61d8ddc4fb51cc0941c96fd1be2076b1", "shape": [ 4, 4 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "6e92b4c838f9ab461fc45d72308d5a5360af4497fff417bae914541608979a02", "shape": [ 4, 5 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 4 }, "source_nodes": [ "s:158" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:129", "inputs": [ "l:105", "l:130" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:159" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "l:130", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "024712320e6f0d6455efd690279f94b44c2b7ac0f5351f626965347a9d3d4199", "shape": [ 4 ], "values": [ 0.5, 0.5, 0.5, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:160" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.5, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:131", "inputs": [ "l:132" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:161" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:132", "inputs": [ "l:112", "l:8", "l:133" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:162" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:133", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:163" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:134", "inputs": [ "l:135", "l:138", "l:140" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:164" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:135", "inputs": [ "l:136", "l:137" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:165" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:136", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:166" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:137", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:167" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:138", "inputs": [ "l:7", "l:139" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:168" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:139", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:169" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:140", "inputs": [ "l:141", "l:159", "l:175" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:170" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:141", "inputs": [ "l:142" ], "name": "collapse_pl_Attn#100_m210_0_9", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "3fdefc9ed00f00dea0599794d1c9a69cee3f0f6849c87508fff2da9d6bbb3ddb", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b05e3b5ab8b2d409e8b3d5f16ea3c8191e3638bbb3168f525a428a2af82fba6b", "shape": [ 9, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "eaa82dd1f86788890259eaf455712f67cccbe5433ad98eac3cad71e322d8d347", "shape": [ 9, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:171" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009574060440063))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:142", "inputs": [ "l:143", "l:146", "l:148" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:178" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:143", "inputs": [ "l:144", "l:145" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:179" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:144", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:180" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:145", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:181" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:146", "inputs": [ "l:7", "l:147" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:182" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:147", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:183" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:148", "inputs": [ "l:149" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:184" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:149", "inputs": [ "l:150" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "86ace85102b9d8a1c0886e9cf2cbb181c89f8ca60093d8ccbe6cbd12a754ad1b", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5870a33a3d94db2db2aaa92ac9edb125e7803a503405c199042a2f0c6d461a6b", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "41d043a7c0f00b8b39f1365727f09ede61e4b865694b6dc1d4acbbd97029d11b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a4d06da2357859d57b942afb3157f9957beb9485b57a8ae112cd6e8f5e6b3764", "shape": [ 32, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "source_nodes": [ "s:185" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6633250767579517, hi=5.503341589908715))" } ], "claimed_range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:150", "inputs": [ "l:151", "l:152", "l:158" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:186" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0021755039062499]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:151", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:187" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:152", "inputs": [ "l:153", "l:156", "l:41" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:188" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:153", "inputs": [ "l:154", "l:155" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:189" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:154", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:190" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:155", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:191" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:156", "inputs": [ "l:41", "l:157" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:192" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:157", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:193" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:158", "inputs": [ "l:41" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:194" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:159", "inputs": [ "l:160", "l:172" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:199" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.5118727628515622, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:160", "inputs": [ "l:161", "l:171" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:200" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.0093727628515623, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:161", "inputs": [ "l:162" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "acc2e4c62e9b92c67550621c1d3c42498eab8b87200286819aae131530c4dc41", "shape": [ 3, 4 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e5cd6f63a143d03bf475cd19b883bbdecec4efb62d8303f209b0dde6ae0565cd", "shape": [ 3, 3 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "ba88de7e97f50c001a9d0c4e5c690cd5e4d90c7e5a7b64b96b485a3847228885", "shape": [ 3, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 3 }, "source_nodes": [ "s:201" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.004351007812499974, hi=1.0043510078125))" } ], "claimed_range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:162", "inputs": [ "l:163", "l:166" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:202" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:163", "inputs": [ "l:164" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f22239942c4ac043eb6887cd996054943f2d24a5e56217c21186a366ca97c23d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9fe7252ae4cff6be4d42fdf46fd404e91cf1503a785d6ee2cfc0779c6e4c7a57", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:203" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:164", "inputs": [ "l:114", "l:116", "l:165" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:205" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:165", "inputs": [ "l:52" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:206" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:166", "inputs": [ "l:167" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "fbfe4ca61ec299680879894a16ca776f81f23454312659ac8736249aa1e9722b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:207" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:167", "inputs": [ "l:168" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "source_nodes": [ "s:209" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:168", "inputs": [ "l:169", "l:170" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:210" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:169", "inputs": [ "l:148" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:211" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:170", "inputs": [ "l:169" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:212" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:171", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:213" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:172", "inputs": [ "l:173" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6ad01752d1a3928862bafa2e6bbe6b2df45d614b086e065fdacf50cb7180f6f3", "shape": [ 4, 5 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c91315e209cc3cd1c12fae1264b1597c61d8ddc4fb51cc0941c96fd1be2076b1", "shape": [ 4, 4 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "6e92b4c838f9ab461fc45d72308d5a5360af4497fff417bae914541608979a02", "shape": [ 4, 5 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 4 }, "source_nodes": [ "s:214" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:173", "inputs": [ "l:41", "l:174" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:215" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "l:174", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "024712320e6f0d6455efd690279f94b44c2b7ac0f5351f626965347a9d3d4199", "shape": [ 4 ], "values": [ 0.5, 0.5, 0.5, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:216" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.5, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:175", "inputs": [ "l:176" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:217" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:176", "inputs": [ "l:163", "l:8", "l:177" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:218" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:177", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:219" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:178", "inputs": [ "l:179" ], "name": "carry_staircase_0_12", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0c8f7b17fb93f1c0ddc6c82c91c7fbe01645654ed1e80bffafdee1b1a61143da", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1493b5cb4e25218f22235b2ef19a32876acac2ec6b2f543bbc7ae76495eb76a8", "shape": [ 12, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f04de261f0526c85e7698c571b8d587ef9557223eee0326a4dad78550febfad3", "shape": [ 12, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "source_nodes": [ "s:220" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=6.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:179", "inputs": [ "l:180", "l:202" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:221" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 33.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:180", "inputs": [ "l:181" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "a17e23a9a0ac633000f4ef1d56a17483b909274a6270009a5915bba0a164881d", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:222" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:181", "inputs": [ "l:80", "l:182", "l:192" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:182", "inputs": [ "l:183" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:226" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:183", "inputs": [ "l:82", "l:184" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:227" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:184", "inputs": [ "l:185", "l:188", "l:190" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:228" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:185", "inputs": [ "l:186", "l:187" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:229" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:186", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:230" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:187", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:231" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:188", "inputs": [ "l:7", "l:189" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:232" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:189", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:233" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:190", "inputs": [ "l:191", "l:159", "l:175" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:234" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:191", "inputs": [ "l:142" ], "name": "collapse_pl_Attn#100_m178_0_13", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "01afa9575aaff88a9f3a51a4578114a24a66192bf8355f33ac56fee5616b18ca", "shape": [ 13 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "fb694bb84d282aed16c503a31849c67f82e5982a0fcf9ac863c4f3998a513076", "shape": [ 13, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "1f400d61c0e178ba2a3d7b83d01a8a931c0d44dc581ca40c6549b6acb82fb385", "shape": [ 13, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 13 }, "source_nodes": [ "s:235" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009574060440063))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:192", "inputs": [ "l:193" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:245" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:193", "inputs": [ "l:194", "l:134" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:246" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:194", "inputs": [ "l:195", "l:198", "l:200" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:247" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:195", "inputs": [ "l:196", "l:197" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:248" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:196", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:249" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:197", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:250" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:198", "inputs": [ "l:7", "l:199" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:251" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:199", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:252" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:200", "inputs": [ "l:201", "l:108", "l:131" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:253" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:201", "inputs": [ "l:90" ], "name": "collapse_pl_Attn#152_m192_0_13", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "01afa9575aaff88a9f3a51a4578114a24a66192bf8355f33ac56fee5616b18ca", "shape": [ 13 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "fb694bb84d282aed16c503a31849c67f82e5982a0fcf9ac863c4f3998a513076", "shape": [ 13, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "1f400d61c0e178ba2a3d7b83d01a8a931c0d44dc581ca40c6549b6acb82fb385", "shape": [ 13, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 13 }, "source_nodes": [ "s:254" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009574060440063))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:202", "inputs": [ "l:203" ], "name": "carry_staircase_0_12", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0c8f7b17fb93f1c0ddc6c82c91c7fbe01645654ed1e80bffafdee1b1a61143da", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1493b5cb4e25218f22235b2ef19a32876acac2ec6b2f543bbc7ae76495eb76a8", "shape": [ 12, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f04de261f0526c85e7698c571b8d587ef9557223eee0326a4dad78550febfad3", "shape": [ 12, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "source_nodes": [ "s:263" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=6.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:203", "inputs": [ "l:204", "l:228" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:264" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 51.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:204", "inputs": [ "l:205" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "5e85f9ba49a0af5e1df5a336fb36c8cb22670d559598896896029f6c7f8b2e71", "shape": [ 10, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:265" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:205", "inputs": [ "l:182", "l:206", "l:192", "l:216", "l:218" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 10 }, { "annotation": null, "id": "l:206", "inputs": [ "l:207" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:269" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:207", "inputs": [ "l:82", "l:208" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:270" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:208", "inputs": [ "l:209", "l:212", "l:214" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:271" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:209", "inputs": [ "l:210", "l:211" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:272" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:210", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:273" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:211", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:274" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:212", "inputs": [ "l:7", "l:213" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:275" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:213", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:276" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:214", "inputs": [ "l:215", "l:159", "l:175" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:277" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:215", "inputs": [ "l:142" ], "name": "collapse_pl_Attn#100_m108_0_17", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "4a02f2d90e882bbcc9baf4d5ee2c3ee11e4452fded3c1c025541ed7749a28172", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ec23f63c1abc506eff7bd523c8de0d39ff8d1026fc605e895a14c263f015a00b", "shape": [ 17, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50829d67c2320f71da81cbc51e04f11877e2884b2ae9b5e562ad1d5f6f7edc93", "shape": [ 17, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 17 }, "source_nodes": [ "s:278" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009575252532958))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009575252532958, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009575252532958, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:216", "inputs": [ "l:217" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:289" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:217", "inputs": [ "l:194", "l:184" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:290" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:218", "inputs": [ "l:219" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:292" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:219", "inputs": [ "l:220", "l:134" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:293" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:220", "inputs": [ "l:221", "l:224", "l:226" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:294" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:221", "inputs": [ "l:222", "l:223" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:295" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:222", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:296" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:223", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:297" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:224", "inputs": [ "l:7", "l:225" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:298" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:225", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:299" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:226", "inputs": [ "l:227", "l:108", "l:131" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:300" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:227", "inputs": [ "l:90" ], "name": "collapse_pl_Attn#152_m160_0_17", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "4a02f2d90e882bbcc9baf4d5ee2c3ee11e4452fded3c1c025541ed7749a28172", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ec23f63c1abc506eff7bd523c8de0d39ff8d1026fc605e895a14c263f015a00b", "shape": [ 17, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50829d67c2320f71da81cbc51e04f11877e2884b2ae9b5e562ad1d5f6f7edc93", "shape": [ 17, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 17 }, "source_nodes": [ "s:301" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009575252532958))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009575252532958, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009575252532958, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:228", "inputs": [ "l:229" ], "name": "carry_staircase_0_12", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0c8f7b17fb93f1c0ddc6c82c91c7fbe01645654ed1e80bffafdee1b1a61143da", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1493b5cb4e25218f22235b2ef19a32876acac2ec6b2f543bbc7ae76495eb76a8", "shape": [ 12, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f04de261f0526c85e7698c571b8d587ef9557223eee0326a4dad78550febfad3", "shape": [ 12, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "source_nodes": [ "s:310" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=6.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:229", "inputs": [ "l:230", "l:236" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:311" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 51.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:230", "inputs": [ "l:231" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d67537e14beb321ce93c69b4a330880e5d3ea3712a2e57015626869e0b444413", "shape": [ 10, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:312" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:231", "inputs": [ "l:206", "l:216", "l:232", "l:218", "l:234" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 10 }, { "annotation": null, "id": "l:232", "inputs": [ "l:233" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:317" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:233", "inputs": [ "l:194", "l:208" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:318" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:234", "inputs": [ "l:235" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:321" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:235", "inputs": [ "l:220", "l:184" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:322" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:236", "inputs": [ "l:237" ], "name": "carry_staircase_0_12", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0c8f7b17fb93f1c0ddc6c82c91c7fbe01645654ed1e80bffafdee1b1a61143da", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1493b5cb4e25218f22235b2ef19a32876acac2ec6b2f543bbc7ae76495eb76a8", "shape": [ 12, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f04de261f0526c85e7698c571b8d587ef9557223eee0326a4dad78550febfad3", "shape": [ 12, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "source_nodes": [ "s:323" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=6.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:237", "inputs": [ "l:238", "l:242" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:324" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 27.013053046875, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:238", "inputs": [ "l:239" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "a906259dd1919a62fdac039d61961e0f375aef08161ea9d5be01e40a036fe21e", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:325" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:239", "inputs": [ "l:232", "l:234", "l:240" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:240", "inputs": [ "l:241" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:330" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:241", "inputs": [ "l:220", "l:208" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:331" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:242", "inputs": [], "name": "collapse_pl_Linear#246_carry_staircase_0_12", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:332" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:243", "inputs": [ "l:244" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:341" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:244", "inputs": [ "l:245", "l:246" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:343" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:245", "inputs": [ "l:31" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:39" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:246", "inputs": [ "l:247" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:344" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:247", "inputs": [ "l:248" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:346" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:248", "inputs": [ "l:249" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:347" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:249", "inputs": [ "l:250", "l:252", "l:254" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:348" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:250", "inputs": [ "l:251" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:349" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:251", "inputs": [ "l:36", "l:59" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:350" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:252", "inputs": [ "l:253" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:351" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:253", "inputs": [ "l:62", "l:68" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:352" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:254", "inputs": [ "l:255" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:353" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:255", "inputs": [ "l:71", "l:256" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:354" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:256", "inputs": [ "l:179" ], "name": "collapse_pl_Add#353_fused_fused_add_const_multiply_const_onehot_lookup_select_0_134", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "93f636d4a182d159f55e8d824f5b99d692697b5534d8c0908b743ad1233f7de5", "shape": [ 134 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "0cba320c895fe5f239fd4b74d5a47ac0c4fee5f799cfc8f13aa74af1c53d4596", "shape": [ 134, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "cceb33ddf65d3da54691f8720d50826e7615926fbfd230e4ef83e02e0c6c6b4e", "shape": [ 134, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 134 }, "source_nodes": [ "s:355" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005149276857477295, hi=1.0005149276843444))" } ], "claimed_range": { "lower": -0.0005149276857477295, "lower_unbounded": false, "upper": 1.0005149276843444, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005149276857477295, "lower_unbounded": false, "upper": 1.0005149276843444, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:257", "inputs": [ "l:258" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9fe7252ae4cff6be4d42fdf46fd404e91cf1503a785d6ee2cfc0779c6e4c7a57", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:363" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:258", "inputs": [ "l:245", "l:246", "l:259" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:365" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:259", "inputs": [ "l:260" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:366" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:260", "inputs": [ "l:261" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:368" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:261", "inputs": [ "l:262" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:369" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:262", "inputs": [ "l:263", "l:275", "l:277" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:370" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:263", "inputs": [ "l:264" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:371" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:264", "inputs": [ "l:36", "l:265" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:372" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:265", "inputs": [ "l:266" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:373" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:266", "inputs": [ "l:267", "l:267", "l:268" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:374" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:267", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:375" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:268", "inputs": [ "l:269" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "558494986e768a8ac9c07f6604a32ad2b92a48d7cf956da8e03c5ed951a41b11", "shape": [ 200, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:376" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:269", "inputs": [ "l:82", "l:134", "l:270" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:377" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:270", "inputs": [ "l:271" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "558494986e768a8ac9c07f6604a32ad2b92a48d7cf956da8e03c5ed951a41b11", "shape": [ 200, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:378" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:271", "inputs": [ "l:194", "l:184", "l:272" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:379" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:272", "inputs": [ "l:273" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "558494986e768a8ac9c07f6604a32ad2b92a48d7cf956da8e03c5ed951a41b11", "shape": [ 200, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:380" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:273", "inputs": [ "l:220", "l:208", "l:274" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:381" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:274", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ], "values": [ 1.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:382" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:275", "inputs": [ "l:276" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:383" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:276", "inputs": [ "l:62", "l:68" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:384" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:277", "inputs": [ "l:278" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:385" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:278", "inputs": [ "l:71", "l:279" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:386" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:279", "inputs": [ "l:203" ], "name": "collapse_pl_Add#328_fused_fused_add_const_multiply_const_onehot_lookup_select_0_206", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f2b95826e5b0f30af72972eaa562d20dbaad9c0976b4acaee3db298ba3c06b41", "shape": [ 206 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "0c6c9ff06d1adba44c2d9297e3922072a29f1fcfc7f6b04117e062a5f7cf25a6", "shape": [ 206, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "82be49beefb66eec4d22ab7b429c7e909360c2708cf7b8609d39257ccc32a8d8", "shape": [ 206, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 206 }, "source_nodes": [ "s:387" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005149277173835658, hi=1.0005292162960004))" } ], "claimed_range": { "lower": -0.0005149277173835658, "lower_unbounded": false, "upper": 1.0005292162960004, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005149277173835658, "lower_unbounded": false, "upper": 1.0005292162960004, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:280", "inputs": [ "l:281" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:395" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:281", "inputs": [ "l:282" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f6bb1294da2f78cd935b01c7656280df5eaa0439e9d97bc03775825a41a508e4", "shape": [ 4, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:396" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 4.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:282", "inputs": [ "l:245", "l:246", "l:259", "l:283" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:397" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:283", "inputs": [ "l:284" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:398" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:284", "inputs": [ "l:285" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:400" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:285", "inputs": [ "l:286" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:401" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:286", "inputs": [ "l:287", "l:290", "l:326" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:402" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:287", "inputs": [ "l:288" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:403" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:288", "inputs": [ "l:36", "l:289" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:404" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:289", "inputs": [ "l:269" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:405" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:290", "inputs": [ "l:291" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:406" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:291", "inputs": [ "l:62", "l:292" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:407" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:292", "inputs": [ "l:293" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "466a4cf2f6f4299d9cc7fd314c658f94d7a2a1dbe112cb598d2ea23fba6d9a08", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:408" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:293", "inputs": [ "l:294", "l:311", "l:313" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:409" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:294", "inputs": [ "l:295" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:410" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:295", "inputs": [ "l:296", "l:82", "l:134" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:411" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:296", "inputs": [ "l:297" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:412" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:297", "inputs": [ "l:298" ], "name": "cmp_score", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9e115a07ab905736a35a795d4e6bc3563c8cb99ceb53c43840d9195037bc779c", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:413" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -13.0565631015625, "lower_unbounded": false, "upper": 13.0565631015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:298", "inputs": [ "l:299", "l:302", "l:303", "l:306", "l:307", "l:310" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:414" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:299", "inputs": [ "l:300" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:415" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:300", "inputs": [ "l:301" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "64e48d62679d2010a89384ca9bac6ee4bcddfdfb5c838dc5aef8b4e8a40fbec8", "shape": [ 100, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:416" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:301", "inputs": [ "l:82", "l:134" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:417" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:302", "inputs": [ "l:300" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1f63f4bc9d1e1abb868bb7c396f4b79db414fdabb2519e660a356486e507089", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:418" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:303", "inputs": [ "l:304" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:419" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:304", "inputs": [ "l:305" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "64e48d62679d2010a89384ca9bac6ee4bcddfdfb5c838dc5aef8b4e8a40fbec8", "shape": [ 100, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:420" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:305", "inputs": [ "l:194", "l:184" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:421" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:306", "inputs": [ "l:304" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1f63f4bc9d1e1abb868bb7c396f4b79db414fdabb2519e660a356486e507089", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:422" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:307", "inputs": [ "l:308" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:423" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:308", "inputs": [ "l:309" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "64e48d62679d2010a89384ca9bac6ee4bcddfdfb5c838dc5aef8b4e8a40fbec8", "shape": [ 100, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:424" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:309", "inputs": [ "l:220", "l:208" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:425" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:310", "inputs": [ "l:308" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1f63f4bc9d1e1abb868bb7c396f4b79db414fdabb2519e660a356486e507089", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:426" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:311", "inputs": [ "l:312" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:427" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:312", "inputs": [ "l:296", "l:134", "l:82" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:428" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:313", "inputs": [ "l:314" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a123e02d91bb9fb43c77f2d4d4881065db3c7f46750fa54f5e5f385685178577", "shape": [ 200, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:429" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:314", "inputs": [ "l:315", "l:317", "l:319" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:430" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:315", "inputs": [ "l:316" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:431" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:316", "inputs": [ "l:296", "l:194", "l:184" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:432" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:317", "inputs": [ "l:318" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:433" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:318", "inputs": [ "l:296", "l:184", "l:194" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:434" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:319", "inputs": [ "l:320" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a123e02d91bb9fb43c77f2d4d4881065db3c7f46750fa54f5e5f385685178577", "shape": [ 200, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:435" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:320", "inputs": [ "l:321", "l:323", "l:325" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:436" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:321", "inputs": [ "l:322" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:437" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:322", "inputs": [ "l:296", "l:220", "l:208" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:438" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:323", "inputs": [ "l:324" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:439" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:324", "inputs": [ "l:296", "l:208", "l:220" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:440" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:325", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ], "values": [ 1.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:441" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:326", "inputs": [ "l:327" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:442" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:327", "inputs": [ "l:71", "l:328" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:443" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:328", "inputs": [ "l:229" ], "name": "collapse_pl_Add#296_fused_fused_add_const_multiply_const_onehot_lookup_select_0_206", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f2b95826e5b0f30af72972eaa562d20dbaad9c0976b4acaee3db298ba3c06b41", "shape": [ 206 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "0c6c9ff06d1adba44c2d9297e3922072a29f1fcfc7f6b04117e062a5f7cf25a6", "shape": [ 206, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "82be49beefb66eec4d22ab7b429c7e909360c2708cf7b8609d39257ccc32a8d8", "shape": [ 206, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 206 }, "source_nodes": [ "s:444" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005149277173835658, hi=1.0005292162960004))" } ], "claimed_range": { "lower": -0.0005149277173835658, "lower_unbounded": false, "upper": 1.0005292162960004, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005149277173835658, "lower_unbounded": false, "upper": 1.0005292162960004, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:329", "inputs": [ "l:330" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "db8d1a556ec9527f63930661298f4b8607cfcecb3c364235751acfef1ec8ed1a", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:452" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:330", "inputs": [ "l:331" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "6ffd58417e23083d07218fa4d272fa091844ae58d9af49caa33e453e2e8dbaf8", "shape": [ 5, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:453" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.010877519531249935, "lower_unbounded": false, "upper": 5.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:331", "inputs": [ "l:245", "l:246", "l:259", "l:283", "l:332" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "l:332", "inputs": [ "l:333" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:457" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:333", "inputs": [ "l:334" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:458" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:334", "inputs": [ "l:335", "l:338", "l:341" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:459" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:335", "inputs": [ "l:336" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:460" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:336", "inputs": [ "l:36", "l:337" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:461" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:337", "inputs": [ "l:271" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:462" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:338", "inputs": [ "l:339" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:463" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:339", "inputs": [ "l:62", "l:340" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:464" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:340", "inputs": [ "l:314" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "466a4cf2f6f4299d9cc7fd314c658f94d7a2a1dbe112cb598d2ea23fba6d9a08", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:465" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:341", "inputs": [ "l:342" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:466" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:342", "inputs": [ "l:71", "l:343" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:467" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:343", "inputs": [ "l:237" ], "name": "collapse_pl_Add#255_fused_fused_add_const_multiply_const_onehot_lookup_select_0_110", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "49be347436898e0a30e5ab54ced913b7a84a9b58ccbc7bd5498e232afabb801a", "shape": [ 110 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "cbc2d46c7b3c1d4ad40b4437ca2e16bc0efbfbefb90bcbfc24fe6469f4a48d5b", "shape": [ 110, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9abc94c2e2c422935e6b1c83caddbbbc4c5b5da8558aab85befb80784a34ab37", "shape": [ 110, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 110 }, "source_nodes": [ "s:468" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00051467796269836, hi=1.0005149276843444))" } ], "claimed_range": { "lower": -0.00051467796269836, "lower_unbounded": false, "upper": 1.0005149276843444, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00051467796269836, "lower_unbounded": false, "upper": 1.0005149276843444, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:344", "inputs": [ "l:345" ], "name": "fused_fused_add_const_multiply_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "ea2845900b5856c9bf354b1aa9761b5aa6888e5ed61738fe9579ca42bc0f6054", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "ea2845900b5856c9bf354b1aa9761b5aa6888e5ed61738fe9579ca42bc0f6054", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:474" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.013053023437500144, "lower_unbounded": false, "upper": 6.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:345", "inputs": [ "l:346" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:477" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:346", "inputs": [ "l:62", "l:347" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:479" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:347", "inputs": [ "l:296" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:480" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:348", "inputs": [ "l:32", "l:248", "l:261", "l:285", "l:333", "l:349", "l:361", "l:361", "l:361", "l:361", "l:361", "l:361" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:482" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:349", "inputs": [ "l:350" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:483" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:350", "inputs": [ "l:351", "l:354", "l:357" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:484" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:351", "inputs": [ "l:352" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:485" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:352", "inputs": [ "l:36", "l:353" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:486" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:353", "inputs": [ "l:273" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:487" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:354", "inputs": [ "l:355" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:488" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:355", "inputs": [ "l:62", "l:356" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:489" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:356", "inputs": [ "l:320" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "466a4cf2f6f4299d9cc7fd314c658f94d7a2a1dbe112cb598d2ea23fba6d9a08", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:490" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:357", "inputs": [ "l:358" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:491" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:358", "inputs": [ "l:71", "l:359" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:492" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:359", "inputs": [ "l:360" ], "name": "collapse_pl_Linear#246_fused_fused_add_const_multiply_const_onehot_lookup_select_0_38", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "561395903c849fb70486027fc90b5815ece800671e87bb25e363d0becf2bff7a", "shape": [ 38 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d7b2c60054ec482022c7c3f61361d1c98c29b9a625cd2bd5ad4a613a08070c4f", "shape": [ 38, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fc59840332071daee887d46bf812287aa3d5230e430d950cd859badb4e6227ea", "shape": [ 38, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 38 }, "source_nodes": [ "s:493" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005037319150666416, hi=1.0005036656926454))" } ], "claimed_range": { "lower": -0.0005037319150666416, "lower_unbounded": false, "upper": 1.0005036656926454, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005037319150666416, "lower_unbounded": false, "upper": 1.0005036656926454, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:360", "inputs": [ "l:240" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:334" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:361", "inputs": [], "name": "remove_leading_0s_sign", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "4ef898bcd05c1b46ff5a572b9c0da67abd133f76ada08aa6e708d5322587027c", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:499" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:362", "inputs": [ "l:363" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:500" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=33.45360234820311))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:363", "inputs": [ "l:364", "l:368" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:501" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:364", "inputs": [ "l:365" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:502" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:365", "inputs": [ "l:366", "l:367" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:504" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:366", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:505" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:367", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9bc792642836b330e7b77ccfcb99d0ea4176f98a05cc12439b285610ead907ae", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:506" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:368", "inputs": [ "l:369" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:508" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:369", "inputs": [ "l:370" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:509" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:370", "inputs": [ "l:26", "l:371" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:510" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:371", "inputs": [ "l:248", "l:261", "l:285", "l:333", "l:349", "l:372", "l:32", "l:248", "l:261", "l:285", "l:333", "l:349" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:511" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:372", "inputs": [ "l:373" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:512" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:373", "inputs": [ "l:374", "l:377", "l:380" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:513" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:374", "inputs": [ "l:375" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:514" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:375", "inputs": [ "l:36", "l:376" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:515" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:376", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "0bbff7d224d8ac368f7ae20a16cb4ecb95004d3ca9ad72eb7433632c0fab6028", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:516" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:377", "inputs": [ "l:378" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:517" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:378", "inputs": [ "l:62", "l:379" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:518" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:379", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "0bbff7d224d8ac368f7ae20a16cb4ecb95004d3ca9ad72eb7433632c0fab6028", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:519" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:380", "inputs": [ "l:381" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:520" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:381", "inputs": [ "l:71", "l:382" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:521" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:382", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "0bbff7d224d8ac368f7ae20a16cb4ecb95004d3ca9ad72eb7433632c0fab6028", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:522" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:383", "inputs": [ "l:384" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:523" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=27.371129193984366))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:384", "inputs": [ "l:385", "l:389" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:524" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:385", "inputs": [ "l:386" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:525" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:386", "inputs": [ "l:387", "l:388" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:527" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:387", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:528" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:388", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ed53dbef9c852a8229e4b386e20627c3a89599c7583ec50379471fb4415bc5b3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:529" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:389", "inputs": [ "l:390" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:531" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:390", "inputs": [ "l:391" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:532" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:391", "inputs": [ "l:26", "l:392" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:533" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:392", "inputs": [ "l:261", "l:285", "l:333", "l:349", "l:372", "l:393", "l:248", "l:261", "l:285", "l:333", "l:349", "l:372" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:534" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:393", "inputs": [ "l:394" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:535" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:394", "inputs": [ "l:395", "l:397", "l:399" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:536" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:395", "inputs": [ "l:396" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:537" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:396", "inputs": [ "l:36", "l:376" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:538" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:397", "inputs": [ "l:398" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:539" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:398", "inputs": [ "l:62", "l:379" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:540" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:399", "inputs": [ "l:400" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:541" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:400", "inputs": [ "l:71", "l:382" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:542" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:401", "inputs": [ "l:402" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:543" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=21.28865603976562))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:402", "inputs": [ "l:403", "l:407" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:544" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:403", "inputs": [ "l:404" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:545" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:404", "inputs": [ "l:405", "l:406" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:547" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:405", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:548" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:406", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9dfa5cc0a4093ad092e8327b9861cf4f1595c50fa4b3f12efc578b78fe5afdd2", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:549" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:407", "inputs": [ "l:408" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:551" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:408", "inputs": [ "l:409" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:552" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:409", "inputs": [ "l:26", "l:410" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:553" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:410", "inputs": [ "l:285", "l:333", "l:349", "l:372", "l:393", "l:393", "l:261", "l:285", "l:333", "l:349", "l:372", "l:393" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:554" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:411", "inputs": [ "l:412" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:555" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=21.288656039765623))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:412", "inputs": [ "l:413", "l:417" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:556" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:413", "inputs": [ "l:414" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:557" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:414", "inputs": [ "l:415", "l:416" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:559" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:415", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:560" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:416", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25eab693bdf7c6ad20a03812dd32b8a7ba27dab587478a5dc1c8b657174ae97d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:561" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:417", "inputs": [ "l:418" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:563" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:418", "inputs": [ "l:419" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:564" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:419", "inputs": [ "l:26", "l:420" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:565" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:420", "inputs": [ "l:333", "l:349", "l:372", "l:393", "l:393", "l:393", "l:285", "l:333", "l:349", "l:372", "l:393", "l:393" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:566" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:421", "inputs": [ "l:422" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:567" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=27.371129193984373))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:422", "inputs": [ "l:423", "l:427" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:568" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:423", "inputs": [ "l:424" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:569" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:424", "inputs": [ "l:425", "l:426" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:571" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:425", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "db8d1a556ec9527f63930661298f4b8607cfcecb3c364235751acfef1ec8ed1a", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:572" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:426", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "64f403ba374325ca540cb27c0375a0ea1dc20ecaeb654fcbe8bea2b756826dab", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:573" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:427", "inputs": [ "l:428" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:575" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:428", "inputs": [ "l:429" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:576" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:429", "inputs": [ "l:26", "l:430" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:577" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:430", "inputs": [ "l:349", "l:372", "l:393", "l:393", "l:393", "l:393", "l:333", "l:349", "l:372", "l:393", "l:393", "l:393" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:578" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:431", "inputs": [ "l:432" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:579" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=33.45360234820312))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:432", "inputs": [ "l:433", "l:437" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:580" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:433", "inputs": [ "l:434" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:581" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:434", "inputs": [ "l:435", "l:436" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:583" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:435", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b8bb7a27e55e02d67b43582292d3134e5715248dc3f4a54dac7e1c71519551b5", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:584" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:436", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df1b487fa89261e6181b4f1197a2248f295f75619fa52d6c77c2a438a9f5862d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:585" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:437", "inputs": [ "l:438" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:587" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:438", "inputs": [ "l:439" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:588" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:439", "inputs": [ "l:26", "l:440" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:589" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:440", "inputs": [ "l:372", "l:393", "l:393", "l:393", "l:393", "l:393", "l:349", "l:372", "l:393", "l:393", "l:393", "l:393" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:590" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:441", "inputs": [ "l:442" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:591" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=36.49483892531249))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:442", "inputs": [ "l:443", "l:447" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:592" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:443", "inputs": [ "l:444" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:593" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:444", "inputs": [ "l:445", "l:446" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:595" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:445", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "fcc789a616d690b13c0ce2bb32111dec233ff36515f19e597b74cfe3b5615bcd", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:596" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:446", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ee1c1e99aec431b3878503f437c2d6ed686bab105f28857153577a62843be75e", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:597" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:447", "inputs": [ "l:448" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:599" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:448", "inputs": [ "l:449" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:600" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:449", "inputs": [ "l:26", "l:450" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:601" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:450", "inputs": [ "l:393", "l:393", "l:393", "l:393", "l:393", "l:393", "l:372", "l:393", "l:393", "l:393", "l:393", "l:393" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:602" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:451", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "bdc7bf3db562e39b656aed4e81ea8425eddc35662c76ad9ef2f006e36d6c43d0", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:603" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 } ], "root": "l:0", "sha256": "9dc230accc48f7f78800dbbdd9b61479fd66f3ac839320625b87ee6a230a6621" }, "realization_map": [ { "lowered": "l:0", "source": "s:0", "status": "whole" }, { "lowered": "l:1", "source": "s:1", "status": "whole" }, { "lowered": "l:2", "source": "s:2", "status": "whole" }, { "lowered": "l:3", "source": "s:3", "status": "whole" }, { "lowered": "l:4", "source": "s:4", "status": "whole" }, { "lowered": "l:5", "source": "s:5", "status": "whole" }, { "lowered": "l:6", "source": "s:6", "status": "whole" }, { "lowered": "l:7", "source": "s:7", "status": "whole" }, { "lowered": "l:8", "source": "s:8", "status": "whole" }, { "lowered": "l:9", "source": "s:9", "status": "whole" }, { "lowered": "l:10", "source": "s:10", "status": "whole" }, { "lowered": "l:11", "source": "s:11", "status": "whole" }, { "lowered": "l:12", "source": "s:12", "status": "whole" }, { "lowered": "l:13", "source": "s:13", "status": "whole" }, { "lowered": "l:14", "source": "s:14", "status": "whole" }, { "source": "s:15", "status": "not_materialized" }, { "lowered": "l:15", "source": "s:16", "status": "whole" }, { "lowered": "l:16", "source": "s:17", "status": "whole" }, { "lowered": "l:17", "source": "s:18", "status": "whole" }, { "lowered": "l:18", "source": "s:19", "status": "whole" }, { "lowered": "l:19", "source": "s:20", "status": "whole" }, { "lowered": "l:20", "source": "s:21", "status": "whole" }, { "lowered": "l:21", "source": "s:22", "status": "whole" }, { "source": "s:23", "status": "not_materialized" }, { "lowered": "l:22", "source": "s:24", "status": "whole" }, { "source": "s:25", "status": "not_materialized" }, { "lowered": "l:23", "source": "s:26", "status": "whole" }, { "lowered": "l:24", "source": "s:27", "status": "whole" }, { "lowered": "l:25", "source": "s:28", "status": "whole" }, { "lowered": "l:26", "source": "s:29", "status": "whole" }, { "source": "s:30", "status": "not_materialized" }, { "lowered": "l:27", "source": "s:31", "status": "whole" }, { "lowered": "l:28", "source": "s:32", "status": "whole" }, { "source": "s:33", "status": "not_materialized" }, { "source": "s:34", "status": "not_materialized" }, { "source": "s:35", "status": "not_materialized" }, { "lowered": "l:30", "source": "s:36", "status": "whole" }, { "source": "s:37", "status": "not_materialized" }, { "source": "s:38", "status": "not_materialized" }, { "lowered": "l:245", "source": "s:39", "status": "whole" }, { "source": "s:40", "status": "not_materialized" }, { "lowered": "l:31", "source": "s:41", "status": "whole" }, { "lowered": "l:32", "source": "s:42", "status": "whole" }, { "lowered": "l:33", "source": "s:43", "status": "whole" }, { "lowered": "l:34", "source": "s:44", "status": "whole" }, { "lowered": "l:35", "source": "s:45", "status": "whole" }, { "lowered": "l:36", "source": "s:46", "status": "whole" }, { "lowered": "l:37", "source": "s:47", "status": "whole" }, { "lowered": "l:38", "source": "s:48", "status": "whole" }, { "lowered": "l:39", "source": "s:49", "status": "whole" }, { "lowered": "l:40", "source": "s:50", "status": "whole" }, { "lowered": "l:41", "source": "s:51", "status": "whole" }, { "source": "s:52", "status": "not_materialized" }, { "lowered": "l:42", "source": "s:53", "status": "whole" }, { "lowered": "l:43", "source": "s:54", "status": "whole" }, { "source": "s:55", "status": "not_materialized" }, { "lowered": "l:44", "source": "s:56", "status": "whole" }, { "lowered": "l:45", "source": "s:57", "status": "whole" }, { "lowered": "l:46", "source": "s:58", "status": "whole" }, { "lowered": "l:47", "source": "s:59", "status": "whole" }, { "lowered": "l:48", "source": "s:60", "status": "whole" }, { "lowered": "l:49", "source": "s:61", "status": "whole" }, { "lowered": "l:50", "source": "s:62", "status": "whole" }, { "lowered": "l:51", "source": "s:63", "status": "whole" }, { "lowered": "l:52", "source": "s:64", "status": "whole" }, { "lowered": "l:53", "source": "s:65", "status": "whole" }, { "lowered": "l:54", "source": "s:66", "status": "whole" }, { "lowered": "l:55", "source": "s:67", "status": "whole" }, { "lowered": "l:56", "source": "s:68", "status": "whole" }, { "lowered": "l:57", "source": "s:69", "status": "whole" }, { "lowered": "l:58", "source": "s:70", "status": "whole" }, { "lowered": "l:59", "source": "s:71", "status": "whole" }, { "lowered": "l:60", "source": "s:72", "status": "whole" }, { "lowered": "l:61", "source": "s:73", "status": "whole" }, { "lowered": "l:62", "source": "s:74", "status": "whole" }, { "lowered": "l:63", "source": "s:75", "status": "whole" }, { "lowered": "l:64", "source": "s:76", "status": "whole" }, { "lowered": "l:65", "source": "s:77", "status": "whole" }, { "lowered": "l:66", "source": "s:78", "status": "whole" }, { "lowered": "l:67", "source": "s:79", "status": "whole" }, { "lowered": "l:68", "source": "s:80", "status": "whole" }, { "lowered": "l:69", "source": "s:81", "status": "whole" }, { "lowered": "l:70", "source": "s:82", "status": "whole" }, { "lowered": "l:71", "source": "s:83", "status": "whole" }, { "lowered": "l:72", "source": "s:84", "status": "whole" }, { "lowered": "l:73", "source": "s:85", "status": "whole" }, { "lowered": "l:74", "source": "s:86", "status": "whole" }, { "lowered": "l:75", "source": "s:87", "status": "whole" }, { "lowered": "l:76", "source": "s:88", "status": "whole" }, { "lowered": "l:77", "source": "s:89", "status": "whole" }, { "source": "s:90", "status": "not_materialized" }, { "source": "s:91", "status": "not_materialized" }, { "source": "s:92", "status": "not_materialized" }, { "source": "s:93", "status": "not_materialized" }, { "lowered": "l:78", "source": "s:94", "status": "whole" }, { "lowered": "l:79", "source": "s:95", "status": "whole" }, { "source": "s:96", "status": "not_materialized" }, { "source": "s:97", "status": "not_materialized" }, { "lowered": "l:80", "source": "s:98", "status": "whole" }, { "lowered": "l:81", "source": "s:99", "status": "whole" }, { "lowered": "l:82", "source": "s:100", "status": "whole" }, { "lowered": "l:83", "source": "s:101", "status": "whole" }, { "lowered": "l:84", "source": "s:102", "status": "whole" }, { "lowered": "l:85", "source": "s:103", "status": "whole" }, { "lowered": "l:86", "source": "s:104", "status": "whole" }, { "lowered": "l:87", "source": "s:105", "status": "whole" }, { "lowered": "l:88", "source": "s:106", "status": "whole" }, { "lowered": "l:89", "source": "s:107", "status": "whole" }, { "source": "s:108", "status": "not_materialized" }, { "source": "s:109", "status": "not_materialized" }, { "source": "s:110", "status": "not_materialized" }, { "source": "s:111", "status": "not_materialized" }, { "source": "s:112", "status": "not_materialized" }, { "source": "s:113", "status": "not_materialized" }, { "lowered": "l:90", "source": "s:114", "status": "whole" }, { "lowered": "l:91", "source": "s:115", "status": "whole" }, { "lowered": "l:92", "source": "s:116", "status": "whole" }, { "lowered": "l:93", "source": "s:117", "status": "whole" }, { "lowered": "l:94", "source": "s:118", "status": "whole" }, { "lowered": "l:95", "source": "s:119", "status": "whole" }, { "lowered": "l:96", "source": "s:120", "status": "whole" }, { "lowered": "l:97", "source": "s:121", "status": "whole" }, { "lowered": "l:98", "source": "s:122", "status": "whole" }, { "lowered": "l:99", "source": "s:123", "status": "whole" }, { "lowered": "l:100", "source": "s:124", "status": "whole" }, { "lowered": "l:101", "source": "s:125", "status": "whole" }, { "lowered": "l:102", "source": "s:126", "status": "whole" }, { "lowered": "l:103", "source": "s:127", "status": "whole" }, { "lowered": "l:104", "source": "s:128", "status": "whole" }, { "lowered": "l:105", "source": "s:129", "status": "whole" }, { "lowered": "l:106", "source": "s:130", "status": "whole" }, { "lowered": "l:107", "source": "s:131", "status": "whole" }, { "source": "s:132", "status": "not_materialized" }, { "source": "s:133", "status": "not_materialized" }, { "source": "s:134", "status": "not_materialized" }, { "source": "s:135", "status": "not_materialized" }, { "lowered": "l:108", "source": "s:136", "status": "whole" }, { "lowered": "l:109", "source": "s:137", "status": "whole" }, { "lowered": "l:110", "source": "s:138", "status": "whole" }, { "lowered": "l:111", "source": "s:139", "status": "whole" }, { "lowered": "l:112", "source": "s:140", "status": "whole" }, { "source": "s:141", "status": "not_materialized" }, { "lowered": "l:113", "source": "s:142", "status": "whole" }, { "lowered": "l:114", "source": "s:143", "status": "whole" }, { "lowered": "l:115", "source": "s:144", "status": "whole" }, { "lowered": "l:116", "source": "s:145", "status": "whole" }, { "lowered": "l:117", "source": "s:146", "status": "whole" }, { "lowered": "l:118", "source": "s:147", "status": "whole" }, { "lowered": "l:119", "source": "s:148", "status": "whole" }, { "lowered": "l:120", "source": "s:149", "status": "whole" }, { "lowered": "l:121", "source": "s:150", "status": "whole" }, { "lowered": "l:122", "source": "s:151", "status": "whole" }, { "source": "s:152", "status": "not_materialized" }, { "lowered": "l:123", "source": "s:153", "status": "whole" }, { "lowered": "l:124", "source": "s:154", "status": "whole" }, { "lowered": "l:125", "source": "s:155", "status": "whole" }, { "lowered": "l:126", "source": "s:156", "status": "whole" }, { "lowered": "l:127", "source": "s:157", "status": "whole" }, { "lowered": "l:128", "source": "s:158", "status": "whole" }, { "lowered": "l:129", "source": "s:159", "status": "whole" }, { "lowered": "l:130", "source": "s:160", "status": "whole" }, { "lowered": "l:131", "source": "s:161", "status": "whole" }, { "lowered": "l:132", "source": "s:162", "status": "whole" }, { "lowered": "l:133", "source": "s:163", "status": "whole" }, { "lowered": "l:134", "source": "s:164", "status": "whole" }, { "lowered": "l:135", "source": "s:165", "status": "whole" }, { "lowered": "l:136", "source": "s:166", "status": "whole" }, { "lowered": "l:137", "source": "s:167", "status": "whole" }, { "lowered": "l:138", "source": "s:168", "status": "whole" }, { "lowered": "l:139", "source": "s:169", "status": "whole" }, { "lowered": "l:140", "source": "s:170", "status": "whole" }, { "lowered": "l:141", "source": "s:171", "status": "whole" }, { "source": "s:172", "status": "not_materialized" }, { "source": "s:173", "status": "not_materialized" }, { "source": "s:174", "status": "not_materialized" }, { "source": "s:175", "status": "not_materialized" }, { "source": "s:176", "status": "not_materialized" }, { "source": "s:177", "status": "not_materialized" }, { "lowered": "l:142", "source": "s:178", "status": "whole" }, { "lowered": "l:143", "source": "s:179", "status": "whole" }, { "lowered": "l:144", "source": "s:180", "status": "whole" }, { "lowered": "l:145", "source": "s:181", "status": "whole" }, { "lowered": "l:146", "source": "s:182", "status": "whole" }, { "lowered": "l:147", "source": "s:183", "status": "whole" }, { "lowered": "l:148", "source": "s:184", "status": "whole" }, { "lowered": "l:149", "source": "s:185", "status": "whole" }, { "lowered": "l:150", "source": "s:186", "status": "whole" }, { "lowered": "l:151", "source": "s:187", "status": "whole" }, { "lowered": "l:152", "source": "s:188", "status": "whole" }, { "lowered": "l:153", "source": "s:189", "status": "whole" }, { "lowered": "l:154", "source": "s:190", "status": "whole" }, { "lowered": "l:155", "source": "s:191", "status": "whole" }, { "lowered": "l:156", "source": "s:192", "status": "whole" }, { "lowered": "l:157", "source": "s:193", "status": "whole" }, { "lowered": "l:158", "source": "s:194", "status": "whole" }, { "source": "s:195", "status": "not_materialized" }, { "source": "s:196", "status": "not_materialized" }, { "source": "s:197", "status": "not_materialized" }, { "source": "s:198", "status": "not_materialized" }, { "lowered": "l:159", "source": "s:199", "status": "whole" }, { "lowered": "l:160", "source": "s:200", "status": "whole" }, { "lowered": "l:161", "source": "s:201", "status": "whole" }, { "lowered": "l:162", "source": "s:202", "status": "whole" }, { "lowered": "l:163", "source": "s:203", "status": "whole" }, { "source": "s:204", "status": "not_materialized" }, { "lowered": "l:164", "source": "s:205", "status": "whole" }, { "lowered": "l:165", "source": "s:206", "status": "whole" }, { "lowered": "l:166", "source": "s:207", "status": "whole" }, { "source": "s:208", "status": "not_materialized" }, { "lowered": "l:167", "source": "s:209", "status": "whole" }, { "lowered": "l:168", "source": "s:210", "status": "whole" }, { "lowered": "l:169", "source": "s:211", "status": "whole" }, { "lowered": "l:170", "source": "s:212", "status": "whole" }, { "lowered": "l:171", "source": "s:213", "status": "whole" }, { "lowered": "l:172", "source": "s:214", "status": "whole" }, { "lowered": "l:173", "source": "s:215", "status": "whole" }, { "lowered": "l:174", "source": "s:216", "status": "whole" }, { "lowered": "l:175", "source": "s:217", "status": "whole" }, { "lowered": "l:176", "source": "s:218", "status": "whole" }, { "lowered": "l:177", "source": "s:219", "status": "whole" }, { "lowered": "l:178", "source": "s:220", "status": "whole" }, { "lowered": "l:179", "source": "s:221", "status": "whole" }, { "lowered": "l:180", "source": "s:222", "status": "whole" }, { "source": "s:223", "status": "not_materialized" }, { "source": "s:224", "status": "not_materialized" }, { "source": "s:225", "status": "not_materialized" }, { "lowered": "l:182", "source": "s:226", "status": "whole" }, { "lowered": "l:183", "source": "s:227", "status": "whole" }, { "lowered": "l:184", "source": "s:228", "status": "whole" }, { "lowered": "l:185", "source": "s:229", "status": "whole" }, { "lowered": "l:186", "source": "s:230", "status": "whole" }, { "lowered": "l:187", "source": "s:231", "status": "whole" }, { "lowered": "l:188", "source": "s:232", "status": "whole" }, { "lowered": "l:189", "source": "s:233", "status": "whole" }, { "lowered": "l:190", "source": "s:234", "status": "whole" }, { "lowered": "l:191", "source": "s:235", "status": "whole" }, { "source": "s:236", "status": "not_materialized" }, { "source": "s:237", "status": "not_materialized" }, { "source": "s:238", "status": "not_materialized" }, { "source": "s:239", "status": "not_materialized" }, { "source": "s:240", "status": "not_materialized" }, { "source": "s:241", "status": "not_materialized" }, { "source": "s:242", "status": "not_materialized" }, { "source": "s:243", "status": "not_materialized" }, { "source": "s:244", "status": "not_materialized" }, { "lowered": "l:192", "source": "s:245", "status": "whole" }, { "lowered": "l:193", "source": "s:246", "status": "whole" }, { "lowered": "l:194", "source": "s:247", "status": "whole" }, { "lowered": "l:195", "source": "s:248", "status": "whole" }, { "lowered": "l:196", "source": "s:249", "status": "whole" }, { "lowered": "l:197", "source": "s:250", "status": "whole" }, { "lowered": "l:198", "source": "s:251", "status": "whole" }, { "lowered": "l:199", "source": "s:252", "status": "whole" }, { "lowered": "l:200", "source": "s:253", "status": "whole" }, { "lowered": "l:201", "source": "s:254", "status": "whole" }, { "source": "s:255", "status": "not_materialized" }, { "source": "s:256", "status": "not_materialized" }, { "source": "s:257", "status": "not_materialized" }, { "source": "s:258", "status": "not_materialized" }, { "source": "s:259", "status": "not_materialized" }, { "source": "s:260", "status": "not_materialized" }, { "source": "s:261", "status": "not_materialized" }, { "source": "s:262", "status": "not_materialized" }, { "lowered": "l:202", "source": "s:263", "status": "whole" }, { "lowered": "l:203", "source": "s:264", "status": "whole" }, { "lowered": "l:204", "source": "s:265", "status": "whole" }, { "source": "s:266", "status": "not_materialized" }, { "source": "s:267", "status": "not_materialized" }, { "source": "s:268", "status": "not_materialized" }, { "lowered": "l:206", "source": "s:269", "status": "whole" }, { "lowered": "l:207", "source": "s:270", "status": "whole" }, { "lowered": "l:208", "source": "s:271", "status": "whole" }, { "lowered": "l:209", "source": "s:272", "status": "whole" }, { "lowered": "l:210", "source": "s:273", "status": "whole" }, { "lowered": "l:211", "source": "s:274", "status": "whole" }, { "lowered": "l:212", "source": "s:275", "status": "whole" }, { "lowered": "l:213", "source": "s:276", "status": "whole" }, { "lowered": "l:214", "source": "s:277", "status": "whole" }, { "lowered": "l:215", "source": "s:278", "status": "whole" }, { "source": "s:279", "status": "not_materialized" }, { "source": "s:280", "status": "not_materialized" }, { "source": "s:281", "status": "not_materialized" }, { "source": "s:282", "status": "not_materialized" }, { "source": "s:283", "status": "not_materialized" }, { "source": "s:284", "status": "not_materialized" }, { "source": "s:285", "status": "not_materialized" }, { "source": "s:286", "status": "not_materialized" }, { "source": "s:287", "status": "not_materialized" }, { "source": "s:288", "status": "not_materialized" }, { "lowered": "l:216", "source": "s:289", "status": "whole" }, { "lowered": "l:217", "source": "s:290", "status": "whole" }, { "source": "s:291", "status": "not_materialized" }, { "lowered": "l:218", "source": "s:292", "status": "whole" }, { "lowered": "l:219", "source": "s:293", "status": "whole" }, { "lowered": "l:220", "source": "s:294", "status": "whole" }, { "lowered": "l:221", "source": "s:295", "status": "whole" }, { "lowered": "l:222", "source": "s:296", "status": "whole" }, { "lowered": "l:223", "source": "s:297", "status": "whole" }, { "lowered": "l:224", "source": "s:298", "status": "whole" }, { "lowered": "l:225", "source": "s:299", "status": "whole" }, { "lowered": "l:226", "source": "s:300", "status": "whole" }, { "lowered": "l:227", "source": "s:301", "status": "whole" }, { "source": "s:302", "status": "not_materialized" }, { "source": "s:303", "status": "not_materialized" }, { "source": "s:304", "status": "not_materialized" }, { "source": "s:305", "status": "not_materialized" }, { "source": "s:306", "status": "not_materialized" }, { "source": "s:307", "status": "not_materialized" }, { "source": "s:308", "status": "not_materialized" }, { "source": "s:309", "status": "not_materialized" }, { "lowered": "l:228", "source": "s:310", "status": "whole" }, { "lowered": "l:229", "source": "s:311", "status": "whole" }, { "lowered": "l:230", "source": "s:312", "status": "whole" }, { "source": "s:313", "status": "not_materialized" }, { "source": "s:314", "status": "not_materialized" }, { "source": "s:315", "status": "not_materialized" }, { "source": "s:316", "status": "not_materialized" }, { "lowered": "l:232", "source": "s:317", "status": "whole" }, { "lowered": "l:233", "source": "s:318", "status": "whole" }, { "source": "s:319", "status": "not_materialized" }, { "source": "s:320", "status": "not_materialized" }, { "lowered": "l:234", "source": "s:321", "status": "whole" }, { "lowered": "l:235", "source": "s:322", "status": "whole" }, { "lowered": "l:236", "source": "s:323", "status": "whole" }, { "lowered": "l:237", "source": "s:324", "status": "whole" }, { "lowered": "l:238", "source": "s:325", "status": "whole" }, { "source": "s:326", "status": "not_materialized" }, { "source": "s:327", "status": "not_materialized" }, { "source": "s:328", "status": "not_materialized" }, { "source": "s:329", "status": "not_materialized" }, { "lowered": "l:240", "source": "s:330", "status": "whole" }, { "lowered": "l:241", "source": "s:331", "status": "whole" }, { "lowered": "l:242", "source": "s:332", "status": "whole" }, { "source": "s:333", "status": "not_materialized" }, { "lowered": "l:360", "source": "s:334", "status": "whole" }, { "source": "s:335", "status": "not_materialized" }, { "source": "s:336", "status": "not_materialized" }, { "source": "s:337", "status": "not_materialized" }, { "source": "s:338", "status": "not_materialized" }, { "source": "s:339", "status": "not_materialized" }, { "source": "s:340", "status": "not_materialized" }, { "lowered": "l:243", "source": "s:341", "status": "whole" }, { "source": "s:342", "status": "not_materialized" }, { "lowered": "l:244", "source": "s:343", "status": "whole" }, { "lowered": "l:246", "source": "s:344", "status": "whole" }, { "source": "s:345", "status": "not_materialized" }, { "lowered": "l:247", "source": "s:346", "status": "whole" }, { "lowered": "l:248", "source": "s:347", "status": "whole" }, { "lowered": "l:249", "source": "s:348", "status": "whole" }, { "lowered": "l:250", "source": "s:349", "status": "whole" }, { "lowered": "l:251", "source": "s:350", "status": "whole" }, { "lowered": "l:252", "source": "s:351", "status": "whole" }, { "lowered": "l:253", "source": "s:352", "status": "whole" }, { "lowered": "l:254", "source": "s:353", "status": "whole" }, { "lowered": "l:255", "source": "s:354", "status": "whole" }, { "lowered": "l:256", "source": "s:355", "status": "whole" }, { "source": "s:356", "status": "not_materialized" }, { "source": "s:357", "status": "not_materialized" }, { "source": "s:358", "status": "not_materialized" }, { "source": "s:359", "status": "not_materialized" }, { "source": "s:360", "status": "not_materialized" }, { "source": "s:361", "status": "not_materialized" }, { "source": "s:362", "status": "not_materialized" }, { "lowered": "l:257", "source": "s:363", "status": "whole" }, { "source": "s:364", "status": "not_materialized" }, { "lowered": "l:258", "source": "s:365", "status": "whole" }, { "lowered": "l:259", "source": "s:366", "status": "whole" }, { "source": "s:367", "status": "not_materialized" }, { "lowered": "l:260", "source": "s:368", "status": "whole" }, { "lowered": "l:261", "source": "s:369", "status": "whole" }, { "lowered": "l:262", "source": "s:370", "status": "whole" }, { "lowered": "l:263", "source": "s:371", "status": "whole" }, { "lowered": "l:264", "source": "s:372", "status": "whole" }, { "lowered": "l:265", "source": "s:373", "status": "whole" }, { "lowered": "l:266", "source": "s:374", "status": "whole" }, { "lowered": "l:267", "source": "s:375", "status": "whole" }, { "lowered": "l:268", "source": "s:376", "status": "whole" }, { "lowered": "l:269", "source": "s:377", "status": "whole" }, { "lowered": "l:270", "source": "s:378", "status": "whole" }, { "lowered": "l:271", "source": "s:379", "status": "whole" }, { "lowered": "l:272", "source": "s:380", "status": "whole" }, { "lowered": "l:273", "source": "s:381", "status": "whole" }, { "lowered": "l:274", "source": "s:382", "status": "whole" }, { "lowered": "l:275", "source": "s:383", "status": "whole" }, { "lowered": "l:276", "source": "s:384", "status": "whole" }, { "lowered": "l:277", "source": "s:385", "status": "whole" }, { "lowered": "l:278", "source": "s:386", "status": "whole" }, { "lowered": "l:279", "source": "s:387", "status": "whole" }, { "source": "s:388", "status": "not_materialized" }, { "source": "s:389", "status": "not_materialized" }, { "source": "s:390", "status": "not_materialized" }, { "source": "s:391", "status": "not_materialized" }, { "source": "s:392", "status": "not_materialized" }, { "source": "s:393", "status": "not_materialized" }, { "source": "s:394", "status": "not_materialized" }, { "lowered": "l:280", "source": "s:395", "status": "whole" }, { "lowered": "l:281", "source": "s:396", "status": "whole" }, { "lowered": "l:282", "source": "s:397", "status": "whole" }, { "lowered": "l:283", "source": "s:398", "status": "whole" }, { "source": "s:399", "status": "not_materialized" }, { "lowered": "l:284", "source": "s:400", "status": "whole" }, { "lowered": "l:285", "source": "s:401", "status": "whole" }, { "lowered": "l:286", "source": "s:402", "status": "whole" }, { "lowered": "l:287", "source": "s:403", "status": "whole" }, { "lowered": "l:288", "source": "s:404", "status": "whole" }, { "lowered": "l:289", "source": "s:405", "status": "whole" }, { "lowered": "l:290", "source": "s:406", "status": "whole" }, { "lowered": "l:291", "source": "s:407", "status": "whole" }, { "lowered": "l:292", "source": "s:408", "status": "whole" }, { "lowered": "l:293", "source": "s:409", "status": "whole" }, { "lowered": "l:294", "source": "s:410", "status": "whole" }, { "lowered": "l:295", "source": "s:411", "status": "whole" }, { "lowered": "l:296", "source": "s:412", "status": "whole" }, { "lowered": "l:297", "source": "s:413", "status": "whole" }, { "lowered": "l:298", "source": "s:414", "status": "whole" }, { "lowered": "l:299", "source": "s:415", "status": "whole" }, { "lowered": "l:300", "source": "s:416", "status": "whole" }, { "lowered": "l:301", "source": "s:417", "status": "whole" }, { "lowered": "l:302", "source": "s:418", "status": "whole" }, { "lowered": "l:303", "source": "s:419", "status": "whole" }, { "lowered": "l:304", "source": "s:420", "status": "whole" }, { "lowered": "l:305", "source": "s:421", "status": "whole" }, { "lowered": "l:306", "source": "s:422", "status": "whole" }, { "lowered": "l:307", "source": "s:423", "status": "whole" }, { "lowered": "l:308", "source": "s:424", "status": "whole" }, { "lowered": "l:309", "source": "s:425", "status": "whole" }, { "lowered": "l:310", "source": "s:426", "status": "whole" }, { "lowered": "l:311", "source": "s:427", "status": "whole" }, { "lowered": "l:312", "source": "s:428", "status": "whole" }, { "lowered": "l:313", "source": "s:429", "status": "whole" }, { "lowered": "l:314", "source": "s:430", "status": "whole" }, { "lowered": "l:315", "source": "s:431", "status": "whole" }, { "lowered": "l:316", "source": "s:432", "status": "whole" }, { "lowered": "l:317", "source": "s:433", "status": "whole" }, { "lowered": "l:318", "source": "s:434", "status": "whole" }, { "lowered": "l:319", "source": "s:435", "status": "whole" }, { "lowered": "l:320", "source": "s:436", "status": "whole" }, { "lowered": "l:321", "source": "s:437", "status": "whole" }, { "lowered": "l:322", "source": "s:438", "status": "whole" }, { "lowered": "l:323", "source": "s:439", "status": "whole" }, { "lowered": "l:324", "source": "s:440", "status": "whole" }, { "lowered": "l:325", "source": "s:441", "status": "whole" }, { "lowered": "l:326", "source": "s:442", "status": "whole" }, { "lowered": "l:327", "source": "s:443", "status": "whole" }, { "lowered": "l:328", "source": "s:444", "status": "whole" }, { "source": "s:445", "status": "not_materialized" }, { "source": "s:446", "status": "not_materialized" }, { "source": "s:447", "status": "not_materialized" }, { "source": "s:448", "status": "not_materialized" }, { "source": "s:449", "status": "not_materialized" }, { "source": "s:450", "status": "not_materialized" }, { "source": "s:451", "status": "not_materialized" }, { "lowered": "l:329", "source": "s:452", "status": "whole" }, { "lowered": "l:330", "source": "s:453", "status": "whole" }, { "source": "s:454", "status": "not_materialized" }, { "source": "s:455", "status": "not_materialized" }, { "source": "s:456", "status": "not_materialized" }, { "lowered": "l:332", "source": "s:457", "status": "whole" }, { "lowered": "l:333", "source": "s:458", "status": "whole" }, { "lowered": "l:334", "source": "s:459", "status": "whole" }, { "lowered": "l:335", "source": "s:460", "status": "whole" }, { "lowered": "l:336", "source": "s:461", "status": "whole" }, { "lowered": "l:337", "source": "s:462", "status": "whole" }, { "lowered": "l:338", "source": "s:463", "status": "whole" }, { "lowered": "l:339", "source": "s:464", "status": "whole" }, { "lowered": "l:340", "source": "s:465", "status": "whole" }, { "lowered": "l:341", "source": "s:466", "status": "whole" }, { "lowered": "l:342", "source": "s:467", "status": "whole" }, { "lowered": "l:343", "source": "s:468", "status": "whole" }, { "source": "s:469", "status": "not_materialized" }, { "source": "s:470", "status": "not_materialized" }, { "source": "s:471", "status": "not_materialized" }, { "source": "s:472", "status": "not_materialized" }, { "source": "s:473", "status": "not_materialized" }, { "lowered": "l:344", "source": "s:474", "status": "whole" }, { "source": "s:475", "status": "not_materialized" }, { "source": "s:476", "status": "not_materialized" }, { "lowered": "l:345", "source": "s:477", "status": "whole" }, { "source": "s:478", "status": "not_materialized" }, { "lowered": "l:346", "source": "s:479", "status": "whole" }, { "lowered": "l:347", "source": "s:480", "status": "whole" }, { "source": "s:481", "status": "not_materialized" }, { "lowered": "l:348", "source": "s:482", "status": "whole" }, { "lowered": "l:349", "source": "s:483", "status": "whole" }, { "lowered": "l:350", "source": "s:484", "status": "whole" }, { "lowered": "l:351", "source": "s:485", "status": "whole" }, { "lowered": "l:352", "source": "s:486", "status": "whole" }, { "lowered": "l:353", "source": "s:487", "status": "whole" }, { "lowered": "l:354", "source": "s:488", "status": "whole" }, { "lowered": "l:355", "source": "s:489", "status": "whole" }, { "lowered": "l:356", "source": "s:490", "status": "whole" }, { "lowered": "l:357", "source": "s:491", "status": "whole" }, { "lowered": "l:358", "source": "s:492", "status": "whole" }, { "lowered": "l:359", "source": "s:493", "status": "whole" }, { "source": "s:494", "status": "not_materialized" }, { "source": "s:495", "status": "not_materialized" }, { "source": "s:496", "status": "not_materialized" }, { "source": "s:497", "status": "not_materialized" }, { "source": "s:498", "status": "not_materialized" }, { "lowered": "l:361", "source": "s:499", "status": "whole" }, { "lowered": "l:362", "source": "s:500", "status": "whole" }, { "lowered": "l:363", "source": "s:501", "status": "whole" }, { "lowered": "l:364", "source": "s:502", "status": "whole" }, { "source": "s:503", "status": "not_materialized" }, { "lowered": "l:365", "source": "s:504", "status": "whole" }, { "lowered": "l:366", "source": "s:505", "status": "whole" }, { "lowered": "l:367", "source": "s:506", "status": "whole" }, { "source": "s:507", "status": "not_materialized" }, { "lowered": "l:368", "source": "s:508", "status": "whole" }, { "lowered": "l:369", "source": "s:509", "status": "whole" }, { "lowered": "l:370", "source": "s:510", "status": "whole" }, { "lowered": "l:371", "source": "s:511", "status": "whole" }, { "lowered": "l:372", "source": "s:512", "status": "whole" }, { "lowered": "l:373", "source": "s:513", "status": "whole" }, { "lowered": "l:374", "source": "s:514", "status": "whole" }, { "lowered": "l:375", "source": "s:515", "status": "whole" }, { "lowered": "l:376", "source": "s:516", "status": "whole" }, { "lowered": "l:377", "source": "s:517", "status": "whole" }, { "lowered": "l:378", "source": "s:518", "status": "whole" }, { "lowered": "l:379", "source": "s:519", "status": "whole" }, { "lowered": "l:380", "source": "s:520", "status": "whole" }, { "lowered": "l:381", "source": "s:521", "status": "whole" }, { "lowered": "l:382", "source": "s:522", "status": "whole" }, { "lowered": "l:383", "source": "s:523", "status": "whole" }, { "lowered": "l:384", "source": "s:524", "status": "whole" }, { "lowered": "l:385", "source": "s:525", "status": "whole" }, { "source": "s:526", "status": "not_materialized" }, { "lowered": "l:386", "source": "s:527", "status": "whole" }, { "lowered": "l:387", "source": "s:528", "status": "whole" }, { "lowered": "l:388", "source": "s:529", "status": "whole" }, { "source": "s:530", "status": "not_materialized" }, { "lowered": "l:389", "source": "s:531", "status": "whole" }, { "lowered": "l:390", "source": "s:532", "status": "whole" }, { "lowered": "l:391", "source": "s:533", "status": "whole" }, { "lowered": "l:392", "source": "s:534", "status": "whole" }, { "lowered": "l:393", "source": "s:535", "status": "whole" }, { "lowered": "l:394", "source": "s:536", "status": "whole" }, { "lowered": "l:395", "source": "s:537", "status": "whole" }, { "lowered": "l:396", "source": "s:538", "status": "whole" }, { "lowered": "l:397", "source": "s:539", "status": "whole" }, { "lowered": "l:398", "source": "s:540", "status": "whole" }, { "lowered": "l:399", "source": "s:541", "status": "whole" }, { "lowered": "l:400", "source": "s:542", "status": "whole" }, { "lowered": "l:401", "source": "s:543", "status": "whole" }, { "lowered": "l:402", "source": "s:544", "status": "whole" }, { "lowered": "l:403", "source": "s:545", "status": "whole" }, { "source": "s:546", "status": "not_materialized" }, { "lowered": "l:404", "source": "s:547", "status": "whole" }, { "lowered": "l:405", "source": "s:548", "status": "whole" }, { "lowered": "l:406", "source": "s:549", "status": "whole" }, { "source": "s:550", "status": "not_materialized" }, { "lowered": "l:407", "source": "s:551", "status": "whole" }, { "lowered": "l:408", "source": "s:552", "status": "whole" }, { "lowered": "l:409", "source": "s:553", "status": "whole" }, { "lowered": "l:410", "source": "s:554", "status": "whole" }, { "lowered": "l:411", "source": "s:555", "status": "whole" }, { "lowered": "l:412", "source": "s:556", "status": "whole" }, { "lowered": "l:413", "source": "s:557", "status": "whole" }, { "source": "s:558", "status": "not_materialized" }, { "lowered": "l:414", "source": "s:559", "status": "whole" }, { "lowered": "l:415", "source": "s:560", "status": "whole" }, { "lowered": "l:416", "source": "s:561", "status": "whole" }, { "source": "s:562", "status": "not_materialized" }, { "lowered": "l:417", "source": "s:563", "status": "whole" }, { "lowered": "l:418", "source": "s:564", "status": "whole" }, { "lowered": "l:419", "source": "s:565", "status": "whole" }, { "lowered": "l:420", "source": "s:566", "status": "whole" }, { "lowered": "l:421", "source": "s:567", "status": "whole" }, { "lowered": "l:422", "source": "s:568", "status": "whole" }, { "lowered": "l:423", "source": "s:569", "status": "whole" }, { "source": "s:570", "status": "not_materialized" }, { "lowered": "l:424", "source": "s:571", "status": "whole" }, { "lowered": "l:425", "source": "s:572", "status": "whole" }, { "lowered": "l:426", "source": "s:573", "status": "whole" }, { "source": "s:574", "status": "not_materialized" }, { "lowered": "l:427", "source": "s:575", "status": "whole" }, { "lowered": "l:428", "source": "s:576", "status": "whole" }, { "lowered": "l:429", "source": "s:577", "status": "whole" }, { "lowered": "l:430", "source": "s:578", "status": "whole" }, { "lowered": "l:431", "source": "s:579", "status": "whole" }, { "lowered": "l:432", "source": "s:580", "status": "whole" }, { "lowered": "l:433", "source": "s:581", "status": "whole" }, { "source": "s:582", "status": "not_materialized" }, { "lowered": "l:434", "source": "s:583", "status": "whole" }, { "lowered": "l:435", "source": "s:584", "status": "whole" }, { "lowered": "l:436", "source": "s:585", "status": "whole" }, { "source": "s:586", "status": "not_materialized" }, { "lowered": "l:437", "source": "s:587", "status": "whole" }, { "lowered": "l:438", "source": "s:588", "status": "whole" }, { "lowered": "l:439", "source": "s:589", "status": "whole" }, { "lowered": "l:440", "source": "s:590", "status": "whole" }, { "lowered": "l:441", "source": "s:591", "status": "whole" }, { "lowered": "l:442", "source": "s:592", "status": "whole" }, { "lowered": "l:443", "source": "s:593", "status": "whole" }, { "source": "s:594", "status": "not_materialized" }, { "lowered": "l:444", "source": "s:595", "status": "whole" }, { "lowered": "l:445", "source": "s:596", "status": "whole" }, { "lowered": "l:446", "source": "s:597", "status": "whole" }, { "source": "s:598", "status": "not_materialized" }, { "lowered": "l:447", "source": "s:599", "status": "whole" }, { "lowered": "l:448", "source": "s:600", "status": "whole" }, { "lowered": "l:449", "source": "s:601", "status": "whole" }, { "lowered": "l:450", "source": "s:602", "status": "whole" }, { "lowered": "l:451", "source": "s:603", "status": "whole" } ], "source": { "nodes": [ { "annotation": null, "id": "s:0", "inputs": [ "s:1" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:2", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=146.70925247975623))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:1", "inputs": [ "s:2", "s:10", "s:603" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:2", "inputs": [ "s:3", "s:6", "s:7" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:3", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:3", "inputs": [ "s:4", "s:5" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:3", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:4", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:3", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:5", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:3", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:6", "inputs": [ "s:7", "s:9" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:3", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:7", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3eb8480c9ef49139f499acac7ad953fae7d8c289cb60b001375681e6691ca7d1", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:5", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:8", "inputs": [], "name": null, "op": "Embedding", "parameters": [ { "dtype": "float32", "name": "table", "sha256": "6bbd3059883cf8b4699c1da2e25ef31211d490f885fcaac1f73475d14b493c32", "shape": [ 18, 17 ] } ], "region": "r:6", "scheduling_predecessors": [], "semantics": { "embedding_width": 17, "input_name": "embedding_input", "vocab_size": 18 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:9", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:3", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:10", "inputs": [ "s:11" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d1eb096a138388c9ee051d92d5ac6d8e55f4754d7e7650310fcab089dad79925", "shape": [ 136, 17 ] } ], "region": "r:7", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:11", "inputs": [ "s:12", "s:500", "s:523", "s:543", "s:555", "s:567", "s:579", "s:591" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:7", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.67731311993905, "upper_unbounded": false } }, "width": 136 }, { "annotation": null, "id": "s:12", "inputs": [ "s:13" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:9", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=18.247419462656246))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:13", "inputs": [ "s:14", "s:26" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:8", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:14", "inputs": [ "s:15" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:12", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:15", "inputs": [ "s:16" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:13", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:16", "inputs": [ "s:17", "s:24" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:13", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:17", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:15", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.9956489921875, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:18", "inputs": [ "s:19" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:17", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3418431447269631, "lower_unbounded": false, "upper": 9.50850981139363, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:19", "inputs": [ "s:20" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6c50dd4f13b1539d4f43be532766dbdf7fb95eaa3941166aa039680e2e300a9d", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6b66e1d03190b21246c3ec82a52fd11b0466bbcf48208c4e3f55fb2eaec6cc20", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "483afcc6ea8b039d16cfd49cb47e300e3ec28b944e85daf9d068c4c79a838894", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "60ace6d30c5997f9dc18846fc3aac46f4e0ed2220c14b4c15fcb4f3b0cb7a036", "shape": [ 32, 1 ] } ], "region": "r:20", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6581568552730369, hi=10.50850981139363))" } ], "claimed_range": { "lower": 0.6581568552730369, "lower_unbounded": false, "upper": 10.50850981139363, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6581568552730369, "lower_unbounded": false, "upper": 10.50850981139363, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:20", "inputs": [ "s:21", "s:2", "s:22" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:21", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:21", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:21", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:22", "inputs": [ "s:23" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:23", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:23", "inputs": [ "s:7" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:24", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:24", "inputs": [ "s:25" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:26", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:25", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:27", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:26", "inputs": [ "s:27" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:27", "inputs": [ "s:28" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:30", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:28", "inputs": [ "s:29", "s:482" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:29", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:29", "inputs": [ "s:30" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ff9beaa3dcaa6fe5b2e7cdb26ebff8cfb3d90db02c236911299cdeedf80443f3", "shape": [ 48 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "cc8cf337ee5ca4135fb2b5843d8c2ecb6b7e65ca5fccad03ef3f84da786c212a", "shape": [ 48, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "81681b33ced1a8f08f6888a042ed160ade67bd948c9a6a0f45f325c25249eec2", "shape": [ 12 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "262fb0b41a348fcbf610dbe142f93f4eae30967bf284fa2438e7550aeaeb59ad", "shape": [ 48, 12 ] } ], "region": "r:32", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 48 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 12 }, { "annotation": null, "id": "s:30", "inputs": [ "s:31", "s:481" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:31", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.023930542968749857, "lower_unbounded": false, "upper": 12.023930542968749, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:31", "inputs": [ "s:32", "s:474" ], "name": null, "op": "Add", "parameters": [], "region": "r:33", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.023930542968749857, "lower_unbounded": false, "upper": 11.023930542968749, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:32", "inputs": [ "s:33" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:34", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.010877519531249935, "lower_unbounded": false, "upper": 5.010877519531249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:33", "inputs": [ "s:34", "s:339", "s:361", "s:393", "s:450" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:34", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:34", "inputs": [ "s:35" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:36", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:35", "inputs": [ "s:36" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:37", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:36", "inputs": [ "s:37" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:39", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:37", "inputs": [ "s:38" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:40", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:38", "inputs": [ "s:39" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:40", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:39", "inputs": [ "s:40" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:42", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:40", "inputs": [ "s:41" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:43", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:41", "inputs": [ "s:42" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:45", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:42", "inputs": [ "s:43" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:47", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:43", "inputs": [ "s:44", "s:72", "s:81" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:47", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:44", "inputs": [ "s:45" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:49", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:45", "inputs": [ "s:46", "s:71" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:48", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:46", "inputs": [ "s:47", "s:50", "s:58" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:50", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:47", "inputs": [ "s:48", "s:49" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:50", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:48", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:50", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:49", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:50", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:50", "inputs": [ "s:51", "s:70" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:50", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:51", "inputs": [ "s:52" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:53", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:52", "inputs": [ "s:53" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:54", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:53", "inputs": [ "s:54", "s:63" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:54", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:54", "inputs": [ "s:55" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:57", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:55", "inputs": [ "s:56" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3, 1 ] } ], "region": "r:58", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.00652651171875, "lower_unbounded": false, "upper": 3.00652651171875, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:56", "inputs": [ "s:57", "s:59", "s:61" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:58", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:57", "inputs": [ "s:58" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:60", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:58", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ef7b1f9e5a2d907c7ff2496a32dc2d4879721c404cc224a31671940ba846c4e2", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:62", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:59", "inputs": [ "s:60" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:64", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:60", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "82c5b2a5f705dd388e12fac162246ad059cfa99089bdd1df5103928849793813", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:66", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:61", "inputs": [ "s:62" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:68", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:62", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2e5d5c3fc5f63a23d29988ab3982c8da9a63a8cd2639d8ca27ab5390529327d9", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:70", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:63", "inputs": [ "s:64" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:73", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:64", "inputs": [ "s:65", "s:68", "s:7" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:74", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:65", "inputs": [ "s:66", "s:67" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:74", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:66", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:74", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:67", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:74", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:68", "inputs": [ "s:7", "s:69" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:74", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:69", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:74", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:70", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:50", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:71", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:75", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:72", "inputs": [ "s:73" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:77", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:73", "inputs": [ "s:74", "s:80" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:76", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:74", "inputs": [ "s:75", "s:78", "s:60" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:78", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:75", "inputs": [ "s:76", "s:77" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:78", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:76", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:78", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:77", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:78", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:78", "inputs": [ "s:51", "s:79" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:78", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:79", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:78", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:80", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:79", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:81", "inputs": [ "s:82" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:81", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:82", "inputs": [ "s:83", "s:89" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:80", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:83", "inputs": [ "s:84", "s:87", "s:62" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:82", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:84", "inputs": [ "s:85", "s:86" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:82", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:85", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:82", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:86", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:82", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:87", "inputs": [ "s:51", "s:88" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:82", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:88", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:82", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:89", "inputs": [ "s:90" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "ad5b5f03dcfecacbf263215872007c593a1de32ee0075aa144808f9740fc9356", "shape": [ 61, 17 ] } ], "region": "r:83", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:90", "inputs": [ "s:91" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:85", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:91", "inputs": [ "s:92" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:86", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:92", "inputs": [ "s:93" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:88", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:93", "inputs": [ "s:94", "s:338" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:87", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 16.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:94", "inputs": [ "s:95", "s:220" ], "name": null, "op": "Add", "parameters": [], "region": "r:89", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 15.002175503906251, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:95", "inputs": [ "s:96" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:90", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:96", "inputs": [ "s:97" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:90", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:97", "inputs": [ "s:98" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:91", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:98", "inputs": [ "s:99" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:93", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:99", "inputs": [ "s:100", "s:164" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:94", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:100", "inputs": [ "s:101", "s:104", "s:106" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:95", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:101", "inputs": [ "s:102", "s:103" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:95", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:102", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:95", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:103", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:95", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:104", "inputs": [ "s:7", "s:105" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:95", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:105", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:95", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:106", "inputs": [ "s:107", "s:136", "s:161" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:97", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:107", "inputs": [ "s:108", "s:134" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:98", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:108", "inputs": [ "s:109" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:100", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:109", "inputs": [ "s:110" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:101", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:110", "inputs": [ "s:111" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:103", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:111", "inputs": [ "s:112", "s:133" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:102", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:112", "inputs": [ "s:113" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "71426d210d52fa91812d0a39251aa75ded92519c3d746b8ced4e5a02ec97960d", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:104", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.3366749232420485, "lower_unbounded": false, "upper": 0.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:113", "inputs": [ "s:114" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:106", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:114", "inputs": [ "s:115", "s:118", "s:120" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:107", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:115", "inputs": [ "s:116", "s:117" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:107", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:116", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:107", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:117", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:107", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:118", "inputs": [ "s:51", "s:119" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:107", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:119", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:107", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:120", "inputs": [ "s:121" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:110", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:121", "inputs": [ "s:122" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "86ace85102b9d8a1c0886e9cf2cbb181c89f8ca60093d8ccbe6cbd12a754ad1b", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5870a33a3d94db2db2aaa92ac9edb125e7803a503405c199042a2f0c6d461a6b", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "41d043a7c0f00b8b39f1365727f09ede61e4b865694b6dc1d4acbbd97029d11b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a4d06da2357859d57b942afb3157f9957beb9485b57a8ae112cd6e8f5e6b3764", "shape": [ 32, 1 ] } ], "region": "r:113", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6633250767579517, hi=5.503341589908715))" } ], "claimed_range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:122", "inputs": [ "s:123", "s:124", "s:131" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:114", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:123", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:114", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:124", "inputs": [ "s:125", "s:128", "s:129" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:115", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:125", "inputs": [ "s:126", "s:127" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:115", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:126", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:115", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:127", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:115", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:128", "inputs": [ "s:129", "s:130" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:115", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:129", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "e79f394c9a536f65c56b29d1e593771c6e332b653623054cfde6b4c6056348bf", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:117", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:130", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:115", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:131", "inputs": [ "s:132" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:119", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:132", "inputs": [ "s:129" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:120", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:133", "inputs": [ "s:112" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:121", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:134", "inputs": [ "s:135" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:122", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:135", "inputs": [ "s:108" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:96", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:136", "inputs": [ "s:137", "s:158" ], "name": null, "op": "Add", "parameters": [], "region": "r:123", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.5118727628515622, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:137", "inputs": [ "s:138", "s:157" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:124", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.0093727628515623, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:138", "inputs": [ "s:139" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "acc2e4c62e9b92c67550621c1d3c42498eab8b87200286819aae131530c4dc41", "shape": [ 3, 4 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e5cd6f63a143d03bf475cd19b883bbdecec4efb62d8303f209b0dde6ae0565cd", "shape": [ 3, 3 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "ba88de7e97f50c001a9d0c4e5c690cd5e4d90c7e5a7b64b96b485a3847228885", "shape": [ 3, 4 ] } ], "region": "r:126", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 3 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.004351007812499974, hi=1.0043510078125))" } ], "claimed_range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:139", "inputs": [ "s:140", "s:151" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:125", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:140", "inputs": [ "s:141" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:129", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:141", "inputs": [ "s:142" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:130", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0478611640624997, "lower_unbounded": false, "upper": 20.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:142", "inputs": [ "s:143", "s:144" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:130", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:143", "inputs": [ "s:8" ], "name": "map_to_table", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "2c34ce1df23b838c5abf2a7f6437cca3d3067ed509ff25f11df6b11b582b51eb", "shape": [ 10 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "eee183eff6e72006bb7ace8c5ae3f42029abaa0004524c553789b6cca053a22c", "shape": [ 10, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e7a2f4b7a886aabf181b94c496dabd00bd3f8da32a01f3da03e6d6868564aed6", "shape": [ 10, 1 ] } ], "region": "r:133", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 10 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-21.0, hi=19.0))" } ], "claimed_range": { "lower": -21.0, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:144", "inputs": [ "s:145" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:136", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:145", "inputs": [ "s:146", "s:149", "s:51" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:137", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:146", "inputs": [ "s:147", "s:148" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:137", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:147", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:137", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:148", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:137", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:149", "inputs": [ "s:51", "s:150" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:137", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:150", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:137", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:151", "inputs": [ "s:152" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:139", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:152", "inputs": [ "s:153" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:140", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:153", "inputs": [ "s:154" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:142", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:154", "inputs": [ "s:155", "s:156" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:141", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:155", "inputs": [ "s:120" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:143", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:156", "inputs": [ "s:155" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:144", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:157", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:145", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:158", "inputs": [ "s:159" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6ad01752d1a3928862bafa2e6bbe6b2df45d614b086e065fdacf50cb7180f6f3", "shape": [ 4, 5 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c91315e209cc3cd1c12fae1264b1597c61d8ddc4fb51cc0941c96fd1be2076b1", "shape": [ 4, 4 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "6e92b4c838f9ab461fc45d72308d5a5360af4497fff417bae914541608979a02", "shape": [ 4, 5 ] } ], "region": "r:147", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 4 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:159", "inputs": [ "s:129", "s:160" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:146", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:160", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "024712320e6f0d6455efd690279f94b44c2b7ac0f5351f626965347a9d3d4199", "shape": [ 4 ], "values": [ 0.5, 0.5, 0.5, 1.0 ] } ], "region": "r:148", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.5, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:161", "inputs": [ "s:162" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:150", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:162", "inputs": [ "s:140", "s:8", "s:163" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:149", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:163", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:151", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:164", "inputs": [ "s:165", "s:168", "s:170" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:152", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:165", "inputs": [ "s:166", "s:167" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:152", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:166", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:152", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:167", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:152", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:168", "inputs": [ "s:7", "s:169" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:152", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:169", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:152", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:170", "inputs": [ "s:171", "s:199", "s:217" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:154", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:171", "inputs": [ "s:172", "s:197" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:155", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:172", "inputs": [ "s:173" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:157", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:173", "inputs": [ "s:174" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:158", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:174", "inputs": [ "s:175" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:160", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:175", "inputs": [ "s:176", "s:196" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:159", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:176", "inputs": [ "s:177" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "71426d210d52fa91812d0a39251aa75ded92519c3d746b8ced4e5a02ec97960d", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:161", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.3366749232420485, "lower_unbounded": false, "upper": 0.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:177", "inputs": [ "s:178" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:163", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:178", "inputs": [ "s:179", "s:182", "s:184" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:164", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:179", "inputs": [ "s:180", "s:181" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:164", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:180", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:164", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:181", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:164", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:182", "inputs": [ "s:7", "s:183" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:164", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:183", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:164", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:184", "inputs": [ "s:185" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:167", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:185", "inputs": [ "s:186" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "86ace85102b9d8a1c0886e9cf2cbb181c89f8ca60093d8ccbe6cbd12a754ad1b", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5870a33a3d94db2db2aaa92ac9edb125e7803a503405c199042a2f0c6d461a6b", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "41d043a7c0f00b8b39f1365727f09ede61e4b865694b6dc1d4acbbd97029d11b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a4d06da2357859d57b942afb3157f9957beb9485b57a8ae112cd6e8f5e6b3764", "shape": [ 32, 1 ] } ], "region": "r:170", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6633250767579517, hi=5.503341589908715))" } ], "claimed_range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:186", "inputs": [ "s:187", "s:188", "s:194" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:171", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0021755039062499]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:187", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:171", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:188", "inputs": [ "s:189", "s:192", "s:51" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:172", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:189", "inputs": [ "s:190", "s:191" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:172", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:190", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:172", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:191", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:172", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:192", "inputs": [ "s:51", "s:193" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:172", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:193", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:172", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:194", "inputs": [ "s:195" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:174", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:195", "inputs": [ "s:51" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:175", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:196", "inputs": [ "s:176" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:176", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:197", "inputs": [ "s:198" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:177", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:198", "inputs": [ "s:172" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:153", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:199", "inputs": [ "s:200", "s:214" ], "name": null, "op": "Add", "parameters": [], "region": "r:178", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.5118727628515622, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:200", "inputs": [ "s:201", "s:213" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:179", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.0093727628515623, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:201", "inputs": [ "s:202" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "acc2e4c62e9b92c67550621c1d3c42498eab8b87200286819aae131530c4dc41", "shape": [ 3, 4 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e5cd6f63a143d03bf475cd19b883bbdecec4efb62d8303f209b0dde6ae0565cd", "shape": [ 3, 3 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "ba88de7e97f50c001a9d0c4e5c690cd5e4d90c7e5a7b64b96b485a3847228885", "shape": [ 3, 4 ] } ], "region": "r:181", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 3 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.004351007812499974, hi=1.0043510078125))" } ], "claimed_range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:202", "inputs": [ "s:203", "s:207" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:180", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:203", "inputs": [ "s:204" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f22239942c4ac043eb6887cd996054943f2d24a5e56217c21186a366ca97c23d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:184", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:204", "inputs": [ "s:205" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3, 1 ] } ], "region": "r:185", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0522121718749995, "lower_unbounded": false, "upper": 21.008702015624998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:205", "inputs": [ "s:143", "s:145", "s:206" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:185", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:206", "inputs": [ "s:64" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:188", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:207", "inputs": [ "s:208" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:190", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:208", "inputs": [ "s:209" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:191", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:209", "inputs": [ "s:210" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:193", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:210", "inputs": [ "s:211", "s:212" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:192", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:211", "inputs": [ "s:184" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:194", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:212", "inputs": [ "s:211" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:195", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:213", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:196", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:214", "inputs": [ "s:215" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6ad01752d1a3928862bafa2e6bbe6b2df45d614b086e065fdacf50cb7180f6f3", "shape": [ 4, 5 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c91315e209cc3cd1c12fae1264b1597c61d8ddc4fb51cc0941c96fd1be2076b1", "shape": [ 4, 4 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "6e92b4c838f9ab461fc45d72308d5a5360af4497fff417bae914541608979a02", "shape": [ 4, 5 ] } ], "region": "r:198", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 4 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:215", "inputs": [ "s:51", "s:216" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:197", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:216", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "024712320e6f0d6455efd690279f94b44c2b7ac0f5351f626965347a9d3d4199", "shape": [ 4 ], "values": [ 0.5, 0.5, 0.5, 1.0 ] } ], "region": "r:199", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.5, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:217", "inputs": [ "s:218" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:201", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:218", "inputs": [ "s:203", "s:8", "s:219" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:200", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:219", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:202", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:220", "inputs": [ "s:221" ], "name": "carry_staircase_0_12", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0c8f7b17fb93f1c0ddc6c82c91c7fbe01645654ed1e80bffafdee1b1a61143da", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1493b5cb4e25218f22235b2ef19a32876acac2ec6b2f543bbc7ae76495eb76a8", "shape": [ 12, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f04de261f0526c85e7698c571b8d587ef9557223eee0326a4dad78550febfad3", "shape": [ 12, 1 ] } ], "region": "r:204", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=6.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:221", "inputs": [ "s:222", "s:263" ], "name": null, "op": "Add", "parameters": [], "region": "r:205", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 33.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:222", "inputs": [ "s:223" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3, 1 ] } ], "region": "r:206", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:223", "inputs": [ "s:224", "s:225", "s:244" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:206", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:224", "inputs": [ "s:98" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:207", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:225", "inputs": [ "s:226" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:208", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:226", "inputs": [ "s:227" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:210", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:227", "inputs": [ "s:100", "s:228" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:211", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:228", "inputs": [ "s:229", "s:232", "s:234" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:212", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:229", "inputs": [ "s:230", "s:231" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:212", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:230", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:212", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:231", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:212", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:232", "inputs": [ "s:7", "s:233" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:212", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:233", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:212", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:234", "inputs": [ "s:235", "s:199", "s:217" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:214", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:235", "inputs": [ "s:236", "s:242" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:215", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:236", "inputs": [ "s:237" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:217", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:237", "inputs": [ "s:238" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:218", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:238", "inputs": [ "s:239" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:220", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:239", "inputs": [ "s:240", "s:241" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:219", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:240", "inputs": [ "s:177" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e4767380eb5e2fc046bce28b8b2a30c81c733be1a56203cd9499066086617f6c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:221", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:241", "inputs": [ "s:240" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:222", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:242", "inputs": [ "s:243" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:223", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:243", "inputs": [ "s:236" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:213", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:244", "inputs": [ "s:245" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:224", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:245", "inputs": [ "s:246" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:226", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:246", "inputs": [ "s:247", "s:164" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:227", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:247", "inputs": [ "s:248", "s:251", "s:253" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:228", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:248", "inputs": [ "s:249", "s:250" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:228", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:249", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:228", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:250", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:228", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:251", "inputs": [ "s:7", "s:252" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:228", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:252", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:228", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:253", "inputs": [ "s:254", "s:136", "s:161" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:230", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:254", "inputs": [ "s:255", "s:261" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:231", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:255", "inputs": [ "s:256" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:233", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:256", "inputs": [ "s:257" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:234", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:257", "inputs": [ "s:258" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:236", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:258", "inputs": [ "s:259", "s:260" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:235", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:259", "inputs": [ "s:113" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e4767380eb5e2fc046bce28b8b2a30c81c733be1a56203cd9499066086617f6c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:237", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:260", "inputs": [ "s:259" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:238", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:261", "inputs": [ "s:262" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:239", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:262", "inputs": [ "s:255" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:229", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:263", "inputs": [ "s:264" ], "name": "carry_staircase_0_12", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0c8f7b17fb93f1c0ddc6c82c91c7fbe01645654ed1e80bffafdee1b1a61143da", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1493b5cb4e25218f22235b2ef19a32876acac2ec6b2f543bbc7ae76495eb76a8", "shape": [ 12, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f04de261f0526c85e7698c571b8d587ef9557223eee0326a4dad78550febfad3", "shape": [ 12, 1 ] } ], "region": "r:241", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=6.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:264", "inputs": [ "s:265", "s:310" ], "name": null, "op": "Add", "parameters": [], "region": "r:242", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 51.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:265", "inputs": [ "s:266" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:243", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:266", "inputs": [ "s:267", "s:268", "s:287", "s:288", "s:291" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:243", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:267", "inputs": [ "s:226" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:244", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:268", "inputs": [ "s:269" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:245", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:269", "inputs": [ "s:270" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:247", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:270", "inputs": [ "s:100", "s:271" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:248", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:271", "inputs": [ "s:272", "s:275", "s:277" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:249", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:272", "inputs": [ "s:273", "s:274" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:249", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:273", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:249", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:274", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:249", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:275", "inputs": [ "s:7", "s:276" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:249", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:276", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:249", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:277", "inputs": [ "s:278", "s:199", "s:217" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:251", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:278", "inputs": [ "s:279", "s:285" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:252", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:279", "inputs": [ "s:280" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:254", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:280", "inputs": [ "s:281" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:255", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:281", "inputs": [ "s:282" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:257", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:282", "inputs": [ "s:283", "s:284" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:256", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:283", "inputs": [ "s:177" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:258", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:284", "inputs": [ "s:283" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:259", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420485, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:285", "inputs": [ "s:286" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:260", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:286", "inputs": [ "s:279" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:250", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:287", "inputs": [ "s:245" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:261", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:288", "inputs": [ "s:289" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:262", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:289", "inputs": [ "s:290" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:264", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:290", "inputs": [ "s:247", "s:228" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:265", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:291", "inputs": [ "s:292" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:266", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:292", "inputs": [ "s:293" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:268", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:293", "inputs": [ "s:294", "s:164" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:269", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:294", "inputs": [ "s:295", "s:298", "s:300" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:270", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:295", "inputs": [ "s:296", "s:297" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:270", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:296", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:270", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:297", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:270", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:298", "inputs": [ "s:7", "s:299" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:270", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:299", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:270", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:300", "inputs": [ "s:301", "s:136", "s:161" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:272", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:301", "inputs": [ "s:302", "s:308" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:273", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:302", "inputs": [ "s:303" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:275", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:303", "inputs": [ "s:304" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:276", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:304", "inputs": [ "s:305" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:278", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:305", "inputs": [ "s:306", "s:307" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:277", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:306", "inputs": [ "s:113" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:279", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:307", "inputs": [ "s:306" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:280", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420485, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:308", "inputs": [ "s:309" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:281", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:309", "inputs": [ "s:302" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:271", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:310", "inputs": [ "s:311" ], "name": "carry_staircase_0_12", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0c8f7b17fb93f1c0ddc6c82c91c7fbe01645654ed1e80bffafdee1b1a61143da", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1493b5cb4e25218f22235b2ef19a32876acac2ec6b2f543bbc7ae76495eb76a8", "shape": [ 12, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f04de261f0526c85e7698c571b8d587ef9557223eee0326a4dad78550febfad3", "shape": [ 12, 1 ] } ], "region": "r:283", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=6.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:311", "inputs": [ "s:312", "s:323" ], "name": null, "op": "Add", "parameters": [], "region": "r:284", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 51.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:312", "inputs": [ "s:313" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:285", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:313", "inputs": [ "s:314", "s:315", "s:316", "s:319", "s:320" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:285", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:314", "inputs": [ "s:269" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:286", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:315", "inputs": [ "s:289" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:287", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:316", "inputs": [ "s:317" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:288", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:317", "inputs": [ "s:318" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:290", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:318", "inputs": [ "s:247", "s:271" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:291", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:319", "inputs": [ "s:292" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:292", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:320", "inputs": [ "s:321" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:293", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:321", "inputs": [ "s:322" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:295", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:322", "inputs": [ "s:294", "s:228" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:296", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:323", "inputs": [ "s:324" ], "name": "carry_staircase_0_12", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0c8f7b17fb93f1c0ddc6c82c91c7fbe01645654ed1e80bffafdee1b1a61143da", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1493b5cb4e25218f22235b2ef19a32876acac2ec6b2f543bbc7ae76495eb76a8", "shape": [ 12, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f04de261f0526c85e7698c571b8d587ef9557223eee0326a4dad78550febfad3", "shape": [ 12, 1 ] } ], "region": "r:298", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=6.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:324", "inputs": [ "s:325", "s:332" ], "name": null, "op": "Add", "parameters": [], "region": "r:299", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 27.013053046875, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:325", "inputs": [ "s:326" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3, 1 ] } ], "region": "r:300", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:326", "inputs": [ "s:327", "s:328", "s:329" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:300", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:327", "inputs": [ "s:317" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:301", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:328", "inputs": [ "s:321" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:302", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:329", "inputs": [ "s:330" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:303", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:330", "inputs": [ "s:331" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:305", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:331", "inputs": [ "s:294", "s:271" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:306", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:332", "inputs": [ "s:333" ], "name": "carry_staircase_0_12", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0c8f7b17fb93f1c0ddc6c82c91c7fbe01645654ed1e80bffafdee1b1a61143da", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1493b5cb4e25218f22235b2ef19a32876acac2ec6b2f543bbc7ae76495eb76a8", "shape": [ 12, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f04de261f0526c85e7698c571b8d587ef9557223eee0326a4dad78550febfad3", "shape": [ 12, 1 ] } ], "region": "r:308", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=6.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 6.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 0.013053046875000002, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:333", "inputs": [ "s:334", "s:337" ], "name": null, "op": "Add", "parameters": [], "region": "r:309", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:334", "inputs": [ "s:335" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:310", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:335", "inputs": [ "s:336" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:310", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:336", "inputs": [ "s:330" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:311", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:337", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:312", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:338", "inputs": [ "s:94" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:313", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 16.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:339", "inputs": [ "s:340" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:315", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:340", "inputs": [ "s:341" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:316", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:341", "inputs": [ "s:342" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:318", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:342", "inputs": [ "s:343" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:319", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:343", "inputs": [ "s:39", "s:344" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:319", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:344", "inputs": [ "s:345" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:321", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:345", "inputs": [ "s:346" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:322", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:346", "inputs": [ "s:347" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:324", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:347", "inputs": [ "s:348" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:326", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:348", "inputs": [ "s:349", "s:351", "s:353" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:326", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:349", "inputs": [ "s:350" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:328", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:350", "inputs": [ "s:46", "s:71" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:327", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:351", "inputs": [ "s:352" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:330", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:352", "inputs": [ "s:74", "s:80" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:329", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:353", "inputs": [ "s:354" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:332", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:354", "inputs": [ "s:83", "s:355" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:331", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:355", "inputs": [ "s:356" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "ad5b5f03dcfecacbf263215872007c593a1de32ee0075aa144808f9740fc9356", "shape": [ 61, 17 ] } ], "region": "r:333", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:356", "inputs": [ "s:357" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:335", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:357", "inputs": [ "s:358" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:336", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:358", "inputs": [ "s:359" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:338", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:359", "inputs": [ "s:221", "s:360" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:337", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 34.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:360", "inputs": [ "s:221" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:339", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 34.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:361", "inputs": [ "s:362" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:341", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:362", "inputs": [ "s:363" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:342", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:363", "inputs": [ "s:364" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:344", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:364", "inputs": [ "s:365" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3, 1 ] } ], "region": "r:345", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.006526511718749961, "lower_unbounded": false, "upper": 3.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:365", "inputs": [ "s:39", "s:344", "s:366" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:345", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:366", "inputs": [ "s:367" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:347", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:367", "inputs": [ "s:368" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:348", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:368", "inputs": [ "s:369" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:350", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:369", "inputs": [ "s:370" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:352", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:370", "inputs": [ "s:371", "s:383", "s:385" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:352", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:371", "inputs": [ "s:372" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:354", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:372", "inputs": [ "s:46", "s:373" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:353", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:373", "inputs": [ "s:374" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:356", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:374", "inputs": [ "s:375", "s:375", "s:376" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:357", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:375", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:358", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:376", "inputs": [ "s:377" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "558494986e768a8ac9c07f6604a32ad2b92a48d7cf956da8e03c5ed951a41b11", "shape": [ 200, 2 ] } ], "region": "r:360", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:377", "inputs": [ "s:100", "s:164", "s:378" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:361", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:378", "inputs": [ "s:379" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "558494986e768a8ac9c07f6604a32ad2b92a48d7cf956da8e03c5ed951a41b11", "shape": [ 200, 2 ] } ], "region": "r:363", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:379", "inputs": [ "s:247", "s:228", "s:380" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:364", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:380", "inputs": [ "s:381" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "558494986e768a8ac9c07f6604a32ad2b92a48d7cf956da8e03c5ed951a41b11", "shape": [ 200, 2 ] } ], "region": "r:366", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:381", "inputs": [ "s:294", "s:271", "s:382" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:367", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:382", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ], "values": [ 1.0, 0.0 ] } ], "region": "r:368", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:383", "inputs": [ "s:384" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:370", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:384", "inputs": [ "s:74", "s:80" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:369", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:385", "inputs": [ "s:386" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:372", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:386", "inputs": [ "s:83", "s:387" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:371", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:387", "inputs": [ "s:388" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "ad5b5f03dcfecacbf263215872007c593a1de32ee0075aa144808f9740fc9356", "shape": [ 61, 17 ] } ], "region": "r:373", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:388", "inputs": [ "s:389" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:375", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:389", "inputs": [ "s:390" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:376", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:390", "inputs": [ "s:391" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:378", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:391", "inputs": [ "s:264", "s:392" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:377", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 52.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:392", "inputs": [ "s:264" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:379", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 52.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:393", "inputs": [ "s:394" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:381", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:394", "inputs": [ "s:395" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:382", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:395", "inputs": [ "s:396" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:384", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:396", "inputs": [ "s:397" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f6bb1294da2f78cd935b01c7656280df5eaa0439e9d97bc03775825a41a508e4", "shape": [ 4, 1 ] } ], "region": "r:385", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 4.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:397", "inputs": [ "s:39", "s:344", "s:366", "s:398" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:385", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:398", "inputs": [ "s:399" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:387", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:399", "inputs": [ "s:400" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:388", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:400", "inputs": [ "s:401" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:390", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:401", "inputs": [ "s:402" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:392", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:402", "inputs": [ "s:403", "s:406", "s:442" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:392", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:403", "inputs": [ "s:404" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:394", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:404", "inputs": [ "s:46", "s:405" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:393", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:405", "inputs": [ "s:377" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:396", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:406", "inputs": [ "s:407" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:398", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:407", "inputs": [ "s:74", "s:408" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:397", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:408", "inputs": [ "s:409" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "466a4cf2f6f4299d9cc7fd314c658f94d7a2a1dbe112cb598d2ea23fba6d9a08", "shape": [ 200, 17 ] } ], "region": "r:400", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:409", "inputs": [ "s:410", "s:427", "s:429" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:401", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:410", "inputs": [ "s:411" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:403", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:411", "inputs": [ "s:412", "s:100", "s:164" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:402", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:412", "inputs": [ "s:413" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:405", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:413", "inputs": [ "s:414" ], "name": "cmp_score", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9e115a07ab905736a35a795d4e6bc3563c8cb99ceb53c43840d9195037bc779c", "shape": [ 6, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -13.0565631015625, "lower_unbounded": false, "upper": 13.0565631015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:414", "inputs": [ "s:415", "s:418", "s:419", "s:422", "s:423", "s:426" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:406", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:415", "inputs": [ "s:416" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:408", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:416", "inputs": [ "s:417" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "64e48d62679d2010a89384ca9bac6ee4bcddfdfb5c838dc5aef8b4e8a40fbec8", "shape": [ 100, 1 ] } ], "region": "r:410", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:417", "inputs": [ "s:100", "s:164" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:411", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:418", "inputs": [ "s:416" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1f63f4bc9d1e1abb868bb7c396f4b79db414fdabb2519e660a356486e507089", "shape": [ 2, 1 ] } ], "region": "r:413", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:419", "inputs": [ "s:420" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:415", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:420", "inputs": [ "s:421" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "64e48d62679d2010a89384ca9bac6ee4bcddfdfb5c838dc5aef8b4e8a40fbec8", "shape": [ 100, 1 ] } ], "region": "r:417", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:421", "inputs": [ "s:247", "s:228" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:418", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:422", "inputs": [ "s:420" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1f63f4bc9d1e1abb868bb7c396f4b79db414fdabb2519e660a356486e507089", "shape": [ 2, 1 ] } ], "region": "r:420", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:423", "inputs": [ "s:424" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:422", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:424", "inputs": [ "s:425" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "64e48d62679d2010a89384ca9bac6ee4bcddfdfb5c838dc5aef8b4e8a40fbec8", "shape": [ 100, 1 ] } ], "region": "r:424", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:425", "inputs": [ "s:294", "s:271" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:425", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:426", "inputs": [ "s:424" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1f63f4bc9d1e1abb868bb7c396f4b79db414fdabb2519e660a356486e507089", "shape": [ 2, 1 ] } ], "region": "r:427", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:427", "inputs": [ "s:428" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:429", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:428", "inputs": [ "s:412", "s:164", "s:100" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:428", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:429", "inputs": [ "s:430" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a123e02d91bb9fb43c77f2d4d4881065db3c7f46750fa54f5e5f385685178577", "shape": [ 200, 2 ] } ], "region": "r:431", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:430", "inputs": [ "s:431", "s:433", "s:435" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:432", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:431", "inputs": [ "s:432" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:434", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:432", "inputs": [ "s:412", "s:247", "s:228" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:433", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:433", "inputs": [ "s:434" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:436", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:434", "inputs": [ "s:412", "s:228", "s:247" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:435", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:435", "inputs": [ "s:436" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a123e02d91bb9fb43c77f2d4d4881065db3c7f46750fa54f5e5f385685178577", "shape": [ 200, 2 ] } ], "region": "r:438", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:436", "inputs": [ "s:437", "s:439", "s:441" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:439", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:437", "inputs": [ "s:438" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:441", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:438", "inputs": [ "s:412", "s:294", "s:271" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:440", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:439", "inputs": [ "s:440" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:443", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:440", "inputs": [ "s:412", "s:271", "s:294" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:442", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:441", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ], "values": [ 1.0, 0.0 ] } ], "region": "r:444", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:442", "inputs": [ "s:443" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:446", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:443", "inputs": [ "s:83", "s:444" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:445", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:444", "inputs": [ "s:445" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "ad5b5f03dcfecacbf263215872007c593a1de32ee0075aa144808f9740fc9356", "shape": [ 61, 17 ] } ], "region": "r:447", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:445", "inputs": [ "s:446" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:449", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:446", "inputs": [ "s:447" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:450", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:447", "inputs": [ "s:448" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:452", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:448", "inputs": [ "s:311", "s:449" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:451", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 52.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:449", "inputs": [ "s:311" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:453", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 52.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:450", "inputs": [ "s:451" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:455", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:451", "inputs": [ "s:452" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:456", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:452", "inputs": [ "s:453" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "db8d1a556ec9527f63930661298f4b8607cfcecb3c364235751acfef1ec8ed1a", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:458", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:453", "inputs": [ "s:454" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:459", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.010877519531249935, "lower_unbounded": false, "upper": 5.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:454", "inputs": [ "s:39", "s:344", "s:366", "s:398", "s:455" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:459", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:455", "inputs": [ "s:456" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:461", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:456", "inputs": [ "s:457" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:462", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:457", "inputs": [ "s:458" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:464", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:458", "inputs": [ "s:459" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:466", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:459", "inputs": [ "s:460", "s:463", "s:466" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:466", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:460", "inputs": [ "s:461" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:468", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:461", "inputs": [ "s:46", "s:462" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:467", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:462", "inputs": [ "s:379" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:470", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:463", "inputs": [ "s:464" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:472", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:464", "inputs": [ "s:74", "s:465" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:471", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:465", "inputs": [ "s:430" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "466a4cf2f6f4299d9cc7fd314c658f94d7a2a1dbe112cb598d2ea23fba6d9a08", "shape": [ 200, 17 ] } ], "region": "r:474", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:466", "inputs": [ "s:467" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:476", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:467", "inputs": [ "s:83", "s:468" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:475", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:468", "inputs": [ "s:469" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "ad5b5f03dcfecacbf263215872007c593a1de32ee0075aa144808f9740fc9356", "shape": [ 61, 17 ] } ], "region": "r:477", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:469", "inputs": [ "s:470" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:479", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:470", "inputs": [ "s:471" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:480", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:471", "inputs": [ "s:472" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:482", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:472", "inputs": [ "s:324", "s:473" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:481", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 28.013053046875, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:473", "inputs": [ "s:324" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:483", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 28.013053046875, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:474", "inputs": [ "s:475" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fedcca07b1ccdacce623cb6d8afdeed0314e8508d763e228871f18d4e0ebb7c4", "shape": [ 1, 1 ] } ], "region": "r:484", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.013053023437499922, "lower_unbounded": false, "upper": 6.013053023437499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:475", "inputs": [ "s:476" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:486", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:476", "inputs": [ "s:477" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:487", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:477", "inputs": [ "s:478" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:490", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:478", "inputs": [ "s:479" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:491", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:479", "inputs": [ "s:74", "s:480" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:491", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:480", "inputs": [ "s:412" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:494", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:481", "inputs": [ "s:31" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:495", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.9760694570312501, "lower_unbounded": false, "upper": 12.023930542968749, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:482", "inputs": [ "s:42", "s:347", "s:369", "s:401", "s:458", "s:483", "s:499", "s:499", "s:499", "s:499", "s:499", "s:499" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:496", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:483", "inputs": [ "s:484" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:498", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:484", "inputs": [ "s:485", "s:488", "s:491" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:498", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:485", "inputs": [ "s:486" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:500", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:486", "inputs": [ "s:46", "s:487" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:499", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:487", "inputs": [ "s:381" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:502", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:488", "inputs": [ "s:489" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:504", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:489", "inputs": [ "s:74", "s:490" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:503", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:490", "inputs": [ "s:436" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "466a4cf2f6f4299d9cc7fd314c658f94d7a2a1dbe112cb598d2ea23fba6d9a08", "shape": [ 200, 17 ] } ], "region": "r:506", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:491", "inputs": [ "s:492" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:508", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:492", "inputs": [ "s:83", "s:493" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:507", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:493", "inputs": [ "s:494" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "ad5b5f03dcfecacbf263215872007c593a1de32ee0075aa144808f9740fc9356", "shape": [ 61, 17 ] } ], "region": "r:509", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:494", "inputs": [ "s:495" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:511", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:495", "inputs": [ "s:496" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:512", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:496", "inputs": [ "s:497" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:514", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:497", "inputs": [ "s:333", "s:498" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:513", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 10.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:498", "inputs": [ "s:333" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:515", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 10.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:499", "inputs": [], "name": "remove_leading_0s_sign", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "4ef898bcd05c1b46ff5a572b9c0da67abd133f76ada08aa6e708d5322587027c", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:516", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:500", "inputs": [ "s:501" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:518", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=33.45360234820311))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:501", "inputs": [ "s:502", "s:508" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:517", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:502", "inputs": [ "s:503" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:521", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:503", "inputs": [ "s:504" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:522", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:504", "inputs": [ "s:505", "s:506" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:522", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:505", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:524", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:506", "inputs": [ "s:507" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9bc792642836b330e7b77ccfcb99d0ea4176f98a05cc12439b285610ead907ae", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:526", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:507", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:527", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:508", "inputs": [ "s:509" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:509", "inputs": [ "s:510" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:529", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:510", "inputs": [ "s:29", "s:511" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:528", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:511", "inputs": [ "s:347", "s:369", "s:401", "s:458", "s:483", "s:512", "s:42", "s:347", "s:369", "s:401", "s:458", "s:483" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:530", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:512", "inputs": [ "s:513" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:532", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:513", "inputs": [ "s:514", "s:517", "s:520" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:532", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:514", "inputs": [ "s:515" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:534", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:515", "inputs": [ "s:46", "s:516" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:533", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:516", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "0bbff7d224d8ac368f7ae20a16cb4ecb95004d3ca9ad72eb7433632c0fab6028", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "region": "r:535", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:517", "inputs": [ "s:518" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:537", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:518", "inputs": [ "s:74", "s:519" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:536", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:519", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "0bbff7d224d8ac368f7ae20a16cb4ecb95004d3ca9ad72eb7433632c0fab6028", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "region": "r:538", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:520", "inputs": [ "s:521" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:540", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:521", "inputs": [ "s:83", "s:522" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:539", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:522", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "0bbff7d224d8ac368f7ae20a16cb4ecb95004d3ca9ad72eb7433632c0fab6028", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "region": "r:541", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:523", "inputs": [ "s:524" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:543", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=27.371129193984366))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:524", "inputs": [ "s:525", "s:531" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:542", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:525", "inputs": [ "s:526" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:546", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:526", "inputs": [ "s:527" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:547", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:527", "inputs": [ "s:528", "s:529" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:547", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:528", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:549", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:529", "inputs": [ "s:530" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ed53dbef9c852a8229e4b386e20627c3a89599c7583ec50379471fb4415bc5b3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:551", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:530", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:552", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:531", "inputs": [ "s:532" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:532", "inputs": [ "s:533" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:554", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:533", "inputs": [ "s:29", "s:534" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:553", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:534", "inputs": [ "s:369", "s:401", "s:458", "s:483", "s:512", "s:535", "s:347", "s:369", "s:401", "s:458", "s:483", "s:512" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:555", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:535", "inputs": [ "s:536" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:557", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:536", "inputs": [ "s:537", "s:539", "s:541" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:557", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:537", "inputs": [ "s:538" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:559", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:538", "inputs": [ "s:46", "s:516" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:558", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:539", "inputs": [ "s:540" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:561", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:540", "inputs": [ "s:74", "s:519" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:560", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:541", "inputs": [ "s:542" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:563", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:542", "inputs": [ "s:83", "s:522" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:562", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:543", "inputs": [ "s:544" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:565", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=21.28865603976562))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:544", "inputs": [ "s:545", "s:551" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:564", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:545", "inputs": [ "s:546" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:568", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:546", "inputs": [ "s:547" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:569", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:547", "inputs": [ "s:548", "s:549" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:569", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:548", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:571", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:549", "inputs": [ "s:550" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9dfa5cc0a4093ad092e8327b9861cf4f1595c50fa4b3f12efc578b78fe5afdd2", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:573", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:550", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:574", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:551", "inputs": [ "s:552" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:552", "inputs": [ "s:553" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:576", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:553", "inputs": [ "s:29", "s:554" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:575", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:554", "inputs": [ "s:401", "s:458", "s:483", "s:512", "s:535", "s:535", "s:369", "s:401", "s:458", "s:483", "s:512", "s:535" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:577", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:555", "inputs": [ "s:556" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:579", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=21.288656039765623))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:556", "inputs": [ "s:557", "s:563" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:578", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:557", "inputs": [ "s:558" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:582", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:558", "inputs": [ "s:559" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:583", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:559", "inputs": [ "s:560", "s:561" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:583", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:560", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:585", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:561", "inputs": [ "s:562" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25eab693bdf7c6ad20a03812dd32b8a7ba27dab587478a5dc1c8b657174ae97d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:587", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:562", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:588", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:563", "inputs": [ "s:564" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:564", "inputs": [ "s:565" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:590", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:565", "inputs": [ "s:29", "s:566" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:589", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:566", "inputs": [ "s:458", "s:483", "s:512", "s:535", "s:535", "s:535", "s:401", "s:458", "s:483", "s:512", "s:535", "s:535" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:591", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:567", "inputs": [ "s:568" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:593", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=27.371129193984373))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:568", "inputs": [ "s:569", "s:575" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:592", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:569", "inputs": [ "s:570" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:596", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:570", "inputs": [ "s:571" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:597", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:571", "inputs": [ "s:572", "s:573" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:597", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:572", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "db8d1a556ec9527f63930661298f4b8607cfcecb3c364235751acfef1ec8ed1a", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:599", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:573", "inputs": [ "s:574" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "64f403ba374325ca540cb27c0375a0ea1dc20ecaeb654fcbe8bea2b756826dab", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:601", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:574", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:602", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:575", "inputs": [ "s:576" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:576", "inputs": [ "s:577" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:604", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:577", "inputs": [ "s:29", "s:578" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:603", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:578", "inputs": [ "s:483", "s:512", "s:535", "s:535", "s:535", "s:535", "s:458", "s:483", "s:512", "s:535", "s:535", "s:535" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:605", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:579", "inputs": [ "s:580" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:607", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=33.45360234820312))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:580", "inputs": [ "s:581", "s:587" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:606", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:581", "inputs": [ "s:582" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:610", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:582", "inputs": [ "s:583" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:611", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:583", "inputs": [ "s:584", "s:585" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:611", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:584", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b8bb7a27e55e02d67b43582292d3134e5715248dc3f4a54dac7e1c71519551b5", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:613", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:585", "inputs": [ "s:586" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df1b487fa89261e6181b4f1197a2248f295f75619fa52d6c77c2a438a9f5862d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:615", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:586", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:616", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:587", "inputs": [ "s:588" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:588", "inputs": [ "s:589" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:618", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:589", "inputs": [ "s:29", "s:590" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:617", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:590", "inputs": [ "s:512", "s:535", "s:535", "s:535", "s:535", "s:535", "s:483", "s:512", "s:535", "s:535", "s:535", "s:535" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:619", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:591", "inputs": [ "s:592" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:621", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=36.49483892531249))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:592", "inputs": [ "s:593", "s:599" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:620", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:593", "inputs": [ "s:594" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:624", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:594", "inputs": [ "s:595" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:625", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:595", "inputs": [ "s:596", "s:597" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:625", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:596", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "fcc789a616d690b13c0ce2bb32111dec233ff36515f19e597b74cfe3b5615bcd", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:627", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:597", "inputs": [ "s:598" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ee1c1e99aec431b3878503f437c2d6ed686bab105f28857153577a62843be75e", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:629", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:598", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:630", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:599", "inputs": [ "s:600" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:600", "inputs": [ "s:601" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:632", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:601", "inputs": [ "s:29", "s:602" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:631", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:602", "inputs": [ "s:535", "s:535", "s:535", "s:535", "s:535", "s:535", "s:512", "s:535", "s:535", "s:535", "s:535", "s:535" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:633", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:603", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "bdc7bf3db562e39b656aed4e81ea8425eddc35662c76ad9ef2f006e36d6c43d0", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0 ] } ], "region": "r:634", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 } ], "root": "s:0", "semantic_regions": [ { "annotation": null, "id": "r:0", "op": "swiglu.sequence_ops.output_sequence", "operands": [ "s:7", "s:26", "s:508", "s:531", "s:551", "s:563", "s:575", "s:587", "s:599" ], "params": { "default_output": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:0" ] }, { "annotation": null, "id": "r:1", "op": "swiglu.map_select.select", "operands": [ "s:2", "s:10", "s:603" ], "params": {}, "parent": "r:0", "results": [ "s:0" ] }, { "annotation": null, "id": "r:2", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:1", "results": [ "s:0" ] }, { "annotation": null, "id": "r:3", "op": "attention_ops.get_prev_value", "operands": [ "s:7", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:0", "results": [ "s:2" ] }, { "annotation": null, "id": "r:4", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:7" ] }, { "annotation": null, "id": "r:5", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:4", "results": [ "s:7" ] }, { "annotation": null, "id": "r:6", "op": "inout_nodes.create_onehot_embedding", "operands": [], "params": { "vocab": [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "+", "-", "*", "\n", " ", "", "" ] }, "parent": null, "results": [ "s:8" ] }, { "annotation": null, "id": "r:7", "op": "linear.sum_nodes", "operands": [ "s:12", "s:500", "s:523", "s:543", "s:555", "s:567", "s:579", "s:591" ], "params": { "max_fanout": null }, "parent": "r:0", "results": [ "s:10" ] }, { "annotation": null, "id": "r:8", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:14", "s:26" ], "params": {}, "parent": "r:0", "results": [ "s:12" ] }, { "annotation": null, "id": "r:9", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:13" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:8", "results": [ "s:12" ] }, { "annotation": null, "id": "r:10", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:17", "s:24" ], "params": {}, "parent": "r:0", "results": [ "s:14" ] }, { "annotation": null, "id": "r:11", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:15" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:10", "results": [ "s:14" ] }, { "annotation": null, "id": "r:12", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:15" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:11", "results": [ "s:14" ] }, { "annotation": null, "id": "r:13", "op": "linear.sum_nodes", "operands": [ "s:17", "s:24" ], "params": { "max_fanout": null }, "parent": "r:10", "results": [ "s:15" ] }, { "annotation": null, "id": "r:14", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -0.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:17" ] }, { "annotation": null, "id": "r:15", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:14", "results": [ "s:17" ] }, { "annotation": null, "id": "r:16", "op": "swiglu.marker_count.count_since_marker", "operands": [ "s:2", "s:22" ], "params": { "max_gap": 9, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:0", "results": [ "s:18" ] }, { "annotation": null, "id": "r:17", "op": "linear.add_const", "operands": [ "s:19" ], "params": { "scalar": -1.0 }, "parent": "r:16", "results": [ "s:18" ] }, { "annotation": null, "id": "r:18", "op": "swiglu.arithmetic_ops.reciprocal", "operands": [ "s:20" ], "params": { "max_value": 1.5, "min_value": 0.09523809523809523, "step": 0.045314900153609824 }, "parent": "r:16", "results": [ "s:19" ] }, { "annotation": null, "id": "r:19", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:20" ], "params": { "breakpoints": [ 0.09523809523809523, 0.10409566750659383, 0.11377703393325501, 0.12435881108912687, 0.1359247412318152, 0.14856635502646182, 0.162383695902762, 0.17748611178045046, 0.19399312042882713, 0.21203535530862097, 0.2317555993814113, 0.2533099150679869, 0.27686887929792725, 0.3026189334243631, 0.33076385868689684, 0.36152638889922667, 0.3951499731239911, 0.43190070228432187, 0.472069414958958, 0.5159739990257997, 0.5639619073686907, 0.6164129075562749, 0.6737420872533829, 0.7364031391492301, 0.8048919513987473, 0.8797505319911436, 0.9615712981024407, 1.0510017643771328, 1.1487496672411774, 1.2555885657992032, 1.3723639636404061, 1.5 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 29.988465456237808, "name": "reciprocal" }, "parent": "r:18", "results": [ "s:19" ] }, { "annotation": null, "id": "r:20", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:20" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 32 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "name": "reciprocal_0_32", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:19", "results": [ "s:19" ] }, { "annotation": null, "id": "r:21", "op": "attention_ops.attend_mean_where", "operands": [ "s:2", "s:22" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:16", "results": [ "s:20" ] }, { "annotation": null, "id": "r:22", "op": "linear.bool_to_01", "operands": [ "s:7" ], "params": {}, "parent": "r:0", "results": [ "s:22" ] }, { "annotation": null, "id": "r:23", "op": "linear.multiply_const", "operands": [ "s:23" ], "params": { "scalar": 0.5 }, "parent": "r:22", "results": [ "s:22" ] }, { "annotation": null, "id": "r:24", "op": "linear.add_const", "operands": [ "s:7" ], "params": { "scalar": 1.0 }, "parent": "r:22", "results": [ "s:23" ] }, { "annotation": null, "id": "r:25", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:25" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -0.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:24" ] }, { "annotation": null, "id": "r:26", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:25" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:25", "results": [ "s:24" ] }, { "annotation": null, "id": "r:27", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:25" ] }, { "annotation": null, "id": "r:28", "op": "swiglu.sequence_ops.remove_leading_0s", "operands": [ "s:8", "s:42", "s:347", "s:369", "s:401", "s:458", "s:483", "s:512", "s:535", "s:477" ], "params": { "max_removals": 5, "sign_token": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:26", "s:508", "s:531", "s:551", "s:563", "s:575", "s:587", "s:599" ] }, { "annotation": null, "id": "r:29", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:482" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:27" ] }, { "annotation": null, "id": "r:30", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:28" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:29", "results": [ "s:27" ] }, { "annotation": null, "id": "r:31", "op": "swiglu.map_select.in_range", "operands": [ "s:31", "s:481" ], "params": { "n_slots": 12 }, "parent": "r:28", "results": [ "s:29" ] }, { "annotation": null, "id": "r:32", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:30" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 48 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 48, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 48, 12 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:31", "results": [ "s:29" ] }, { "annotation": null, "id": "r:33", "op": "linear.add", "operands": [ "s:32", "s:474" ], "params": {}, "parent": "r:28", "results": [ "s:31" ] }, { "annotation": null, "id": "r:34", "op": "linear.sum_nodes", "operands": [ "s:34", "s:339", "s:361", "s:393", "s:450" ], "params": { "max_fanout": null }, "parent": "r:28", "results": [ "s:32" ] }, { "annotation": null, "id": "r:35", "op": "linear.bool_to_01", "operands": [ "s:36" ], "params": {}, "parent": "r:28", "results": [ "s:34" ] }, { "annotation": null, "id": "r:36", "op": "linear.multiply_const", "operands": [ "s:35" ], "params": { "scalar": 0.5 }, "parent": "r:35", "results": [ "s:34" ] }, { "annotation": null, "id": "r:37", "op": "linear.add_const", "operands": [ "s:36" ], "params": { "scalar": 1.0 }, "parent": "r:35", "results": [ "s:35" ] }, { "annotation": null, "id": "r:38", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:37" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": "r:28", "results": [ "s:36" ] }, { "annotation": null, "id": "r:39", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:37" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:38", "results": [ "s:36" ] }, { "annotation": null, "id": "r:40", "op": "linear.sum_nodes", "operands": [ "s:39" ], "params": { "max_fanout": null }, "parent": "r:28", "results": [ "s:37" ] }, { "annotation": null, "id": "r:41", "op": "linear.bool_to_01", "operands": [ "s:41" ], "params": {}, "parent": "r:28", "results": [ "s:39" ] }, { "annotation": null, "id": "r:42", "op": "linear.multiply_const", "operands": [ "s:40" ], "params": { "scalar": 0.5 }, "parent": "r:41", "results": [ "s:39" ] }, { "annotation": null, "id": "r:43", "op": "linear.add_const", "operands": [ "s:41" ], "params": { "scalar": 1.0 }, "parent": "r:41", "results": [ "s:40" ] }, { "annotation": null, "id": "r:44", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:42" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:28", "results": [ "s:41" ] }, { "annotation": null, "id": "r:45", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:42" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:44", "results": [ "s:41" ] }, { "annotation": null, "id": "r:46", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:71", "s:80", "s:89" ], "params": {}, "parent": null, "results": [ "s:42" ] }, { "annotation": null, "id": "r:47", "op": "linear.sum_nodes", "operands": [ "s:44", "s:72", "s:81" ], "params": { "max_fanout": null }, "parent": "r:46", "results": [ "s:42" ] }, { "annotation": null, "id": "r:48", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:71" ], "params": {}, "parent": "r:46", "results": [ "s:44" ] }, { "annotation": null, "id": "r:49", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:45" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:48", "results": [ "s:44" ] }, { "annotation": null, "id": "r:50", "op": "attention_ops.get_prev_value", "operands": [ "s:58", "s:51" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:46" ] }, { "annotation": null, "id": "r:51", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:54", "s:63" ], "params": {}, "parent": null, "results": [ "s:51" ] }, { "annotation": null, "id": "r:52", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:52" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:51", "results": [ "s:51" ] }, { "annotation": null, "id": "r:53", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:52" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:52", "results": [ "s:51" ] }, { "annotation": null, "id": "r:54", "op": "linear.sum_nodes", "operands": [ "s:54", "s:63" ], "params": { "max_fanout": null }, "parent": "r:51", "results": [ "s:52" ] }, { "annotation": null, "id": "r:55", "op": "swiglu.logic_ops.bool_any_true", "operands": [ "s:58", "s:60", "s:62" ], "params": {}, "parent": null, "results": [ "s:54" ] }, { "annotation": null, "id": "r:56", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:55" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": "r:55", "results": [ "s:54" ] }, { "annotation": null, "id": "r:57", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:55" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:56", "results": [ "s:54" ] }, { "annotation": null, "id": "r:58", "op": "linear.sum_nodes", "operands": [ "s:57", "s:59", "s:61" ], "params": { "max_fanout": null }, "parent": "r:55", "results": [ "s:55" ] }, { "annotation": null, "id": "r:59", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:58" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": "r:55", "results": [ "s:57" ] }, { "annotation": null, "id": "r:60", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:58" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:59", "results": [ "s:57" ] }, { "annotation": null, "id": "r:61", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:58" ] }, { "annotation": null, "id": "r:62", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:61", "results": [ "s:58" ] }, { "annotation": null, "id": "r:63", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:60" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": "r:55", "results": [ "s:59" ] }, { "annotation": null, "id": "r:64", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:60" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:63", "results": [ "s:59" ] }, { "annotation": null, "id": "r:65", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:60" ] }, { "annotation": null, "id": "r:66", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:65", "results": [ "s:60" ] }, { "annotation": null, "id": "r:67", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:62" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": "r:55", "results": [ "s:61" ] }, { "annotation": null, "id": "r:68", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:62" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:67", "results": [ "s:61" ] }, { "annotation": null, "id": "r:69", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:62" ] }, { "annotation": null, "id": "r:70", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:69", "results": [ "s:62" ] }, { "annotation": null, "id": "r:71", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:64" ], "params": {}, "parent": null, "results": [ "s:63" ] }, { "annotation": null, "id": "r:72", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:64" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:71", "results": [ "s:63" ] }, { "annotation": null, "id": "r:73", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:64" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:72", "results": [ "s:63" ] }, { "annotation": null, "id": "r:74", "op": "attention_ops.get_prev_value", "operands": [ "s:7", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:64" ] }, { "annotation": null, "id": "r:75", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:71" ] }, { "annotation": null, "id": "r:76", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:80" ], "params": {}, "parent": "r:46", "results": [ "s:72" ] }, { "annotation": null, "id": "r:77", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:73" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:76", "results": [ "s:72" ] }, { "annotation": null, "id": "r:78", "op": "attention_ops.get_prev_value", "operands": [ "s:60", "s:51" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:74" ] }, { "annotation": null, "id": "r:79", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:80" ] }, { "annotation": null, "id": "r:80", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:89" ], "params": {}, "parent": "r:46", "results": [ "s:81" ] }, { "annotation": null, "id": "r:81", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:82" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:80", "results": [ "s:81" ] }, { "annotation": null, "id": "r:82", "op": "attention_ops.get_prev_value", "operands": [ "s:62", "s:51" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:83" ] }, { "annotation": null, "id": "r:83", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:90" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:89" ] }, { "annotation": null, "id": "r:84", "op": "linear.bool_to_01", "operands": [ "s:92" ], "params": {}, "parent": null, "results": [ "s:90" ] }, { "annotation": null, "id": "r:85", "op": "linear.multiply_const", "operands": [ "s:91" ], "params": { "scalar": 0.5 }, "parent": "r:84", "results": [ "s:90" ] }, { "annotation": null, "id": "r:86", "op": "linear.add_const", "operands": [ "s:92" ], "params": { "scalar": 1.0 }, "parent": "r:84", "results": [ "s:91" ] }, { "annotation": null, "id": "r:87", "op": "swiglu.map_select.in_range", "operands": [ "s:94", "s:338" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:92" ] }, { "annotation": null, "id": "r:88", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:93" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:87", "results": [ "s:92" ] }, { "annotation": null, "id": "r:89", "op": "linear.add", "operands": [ "s:95", "s:220" ], "params": {}, "parent": null, "results": [ "s:94" ] }, { "annotation": null, "id": "r:90", "op": "linear.sum_nodes", "operands": [ "s:97" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:95" ] }, { "annotation": null, "id": "r:91", "op": "linear.slice_columns", "operands": [ "s:98" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:97" ] }, { "annotation": null, "id": "r:92", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:99" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:98" ] }, { "annotation": null, "id": "r:93", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:99" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:92", "results": [ "s:98" ] }, { "annotation": null, "id": "r:94", "op": "linear.concat", "operands": [ "s:100", "s:164" ], "params": {}, "parent": null, "results": [ "s:99" ] }, { "annotation": null, "id": "r:95", "op": "attention_ops.get_prev_value", "operands": [ "s:106", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:100" ] }, { "annotation": null, "id": "r:96", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:112" ], "params": {}, "parent": null, "results": [ "s:106" ] }, { "annotation": null, "id": "r:97", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:107", "s:136", "s:161" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:96", "results": [ "s:106" ] }, { "annotation": null, "id": "r:98", "op": "linear.concat", "operands": [ "s:108", "s:134" ], "params": {}, "parent": "r:96", "results": [ "s:107" ] }, { "annotation": null, "id": "r:99", "op": "linear.bool_to_01", "operands": [ "s:110" ], "params": {}, "parent": "r:96", "results": [ "s:108" ] }, { "annotation": null, "id": "r:100", "op": "linear.multiply_const", "operands": [ "s:109" ], "params": { "scalar": 0.5 }, "parent": "r:99", "results": [ "s:108" ] }, { "annotation": null, "id": "r:101", "op": "linear.add_const", "operands": [ "s:110" ], "params": { "scalar": 1.0 }, "parent": "r:99", "results": [ "s:109" ] }, { "annotation": null, "id": "r:102", "op": "swiglu.map_select.in_range", "operands": [ "s:112", "s:133" ], "params": { "n_slots": 3 }, "parent": "r:96", "results": [ "s:110" ] }, { "annotation": null, "id": "r:103", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:111" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:102", "results": [ "s:110" ] }, { "annotation": null, "id": "r:104", "op": "linear.add_const", "operands": [ "s:113" ], "params": { "scalar": -3.0 }, "parent": null, "results": [ "s:112" ] }, { "annotation": null, "id": "r:105", "op": "swiglu.sequence_ops.IndexedRegion.length_at", "operands": [ "s:51" ], "params": {}, "parent": null, "results": [ "s:113" ] }, { "annotation": null, "id": "r:106", "op": "linear.add_const", "operands": [ "s:114" ], "params": { "scalar": -1.0 }, "parent": "r:105", "results": [ "s:113" ] }, { "annotation": null, "id": "r:107", "op": "attention_ops.get_prev_value", "operands": [ "s:120", "s:51" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:105", "results": [ "s:114" ] }, { "annotation": null, "id": "r:108", "op": "swiglu.sequence_ops.IndexedRegion.__init__", "operands": [ "s:8", "s:129", "s:140" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "max_len": 3, "max_read_distance": 10, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [] }, { "annotation": null, "id": "r:109", "op": "swiglu.marker_count.count_since_marker", "operands": [ "s:124", "s:131" ], "params": { "max_gap": 4, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:108", "results": [ "s:120" ] }, { "annotation": null, "id": "r:110", "op": "linear.add_const", "operands": [ "s:121" ], "params": { "scalar": -1.0 }, "parent": "r:109", "results": [ "s:120" ] }, { "annotation": null, "id": "r:111", "op": "swiglu.arithmetic_ops.reciprocal", "operands": [ "s:122" ], "params": { "max_value": 1.5, "min_value": 0.18181818181818182, "step": 0.04252199413489736 }, "parent": "r:109", "results": [ "s:121" ] }, { "annotation": null, "id": "r:112", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:122" ], "params": { "breakpoints": [ 0.18181818181818182, 0.19462576879929028, 0.20833554434393137, 0.2230110601738595, 0.23872034470395204, 0.2555362183882104, 0.2735366312792783, 0.2928050243662364, 0.3134307163656553, 0.33550931775887877, 0.35914317399480933, 0.3844418399126637, 0.4115225875838872, 0.44051094992733025, 0.4715413026176177, 0.5047574869841495, 0.5403134767881832, 0.5783740919688469, 0.6191157626666555, 0.6627273470661621, 0.7094110068488624, 0.7593831443145113, 0.8128754055148878, 0.8701357540500343, 0.9314296205045596, 0.997041132852221, 1.0672744335323334, 1.142455089303316, 1.2229316004087583, 1.3090770160517557, 1.4012906636660458, 1.5 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 20.739660046174524, "name": "reciprocal" }, "parent": "r:111", "results": [ "s:121" ] }, { "annotation": null, "id": "r:113", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:122" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 32 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "name": "reciprocal_0_32", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:112", "results": [ "s:121" ] }, { "annotation": null, "id": "r:114", "op": "attention_ops.attend_mean_where", "operands": [ "s:124", "s:131" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:109", "results": [ "s:122" ] }, { "annotation": null, "id": "r:115", "op": "attention_ops.get_prev_value", "operands": [ "s:129", "s:129" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:108", "results": [ "s:124" ] }, { "annotation": null, "id": "r:116", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:129" ] }, { "annotation": null, "id": "r:117", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:116", "results": [ "s:129" ] }, { "annotation": null, "id": "r:118", "op": "linear.bool_to_01", "operands": [ "s:129" ], "params": {}, "parent": "r:108", "results": [ "s:131" ] }, { "annotation": null, "id": "r:119", "op": "linear.multiply_const", "operands": [ "s:132" ], "params": { "scalar": 0.5 }, "parent": "r:118", "results": [ "s:131" ] }, { "annotation": null, "id": "r:120", "op": "linear.add_const", "operands": [ "s:129" ], "params": { "scalar": 1.0 }, "parent": "r:118", "results": [ "s:132" ] }, { "annotation": null, "id": "r:121", "op": "linear.add_const", "operands": [ "s:112" ], "params": { "scalar": 1.0 }, "parent": "r:96", "results": [ "s:133" ] }, { "annotation": null, "id": "r:122", "op": "linear.add_const", "operands": [ "s:135" ], "params": { "scalar": 1.0 }, "parent": "r:96", "results": [ "s:134" ] }, { "annotation": null, "id": "r:123", "op": "linear.add", "operands": [ "s:137", "s:158" ], "params": {}, "parent": "r:108", "results": [ "s:136" ] }, { "annotation": null, "id": "r:124", "op": "linear.concat", "operands": [ "s:138", "s:157" ], "params": {}, "parent": "r:108", "results": [ "s:137" ] }, { "annotation": null, "id": "r:125", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:140", "s:151" ], "params": {}, "parent": "r:108", "results": [ "s:138" ] }, { "annotation": null, "id": "r:126", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:139" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 4 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 3 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 4 ] } } }, "parent": "r:125", "results": [ "s:138" ] }, { "annotation": null, "id": "r:127", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:143", "s:144" ], "params": {}, "parent": null, "results": [ "s:140" ] }, { "annotation": null, "id": "r:128", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:141" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:127", "results": [ "s:140" ] }, { "annotation": null, "id": "r:129", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:141" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:128", "results": [ "s:140" ] }, { "annotation": null, "id": "r:130", "op": "linear.sum_nodes", "operands": [ "s:143", "s:144" ], "params": { "max_fanout": null }, "parent": "r:127", "results": [ "s:141" ] }, { "annotation": null, "id": "r:131", "op": "swiglu.sequence_ops.check_is_digit", "operands": [ "s:8" ], "params": {}, "parent": null, "results": [ "s:143" ] }, { "annotation": null, "id": "r:132", "op": "swiglu.map_select.map_to_table", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": "r:131", "results": [ "s:143" ] }, { "annotation": null, "id": "r:133", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 10 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 10, 17 ] } }, "name": "map_to_table", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 10, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:132", "results": [ "s:143" ] }, { "annotation": null, "id": "r:134", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:145" ], "params": {}, "parent": null, "results": [ "s:144" ] }, { "annotation": null, "id": "r:135", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:145" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:134", "results": [ "s:144" ] }, { "annotation": null, "id": "r:136", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:145" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:135", "results": [ "s:144" ] }, { "annotation": null, "id": "r:137", "op": "attention_ops.get_prev_value", "operands": [ "s:51", "s:51" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:145" ] }, { "annotation": null, "id": "r:138", "op": "linear.bool_to_01", "operands": [ "s:153" ], "params": {}, "parent": "r:108", "results": [ "s:151" ] }, { "annotation": null, "id": "r:139", "op": "linear.multiply_const", "operands": [ "s:152" ], "params": { "scalar": 0.5 }, "parent": "r:138", "results": [ "s:151" ] }, { "annotation": null, "id": "r:140", "op": "linear.add_const", "operands": [ "s:153" ], "params": { "scalar": 1.0 }, "parent": "r:138", "results": [ "s:152" ] }, { "annotation": null, "id": "r:141", "op": "swiglu.map_select.in_range", "operands": [ "s:155", "s:156" ], "params": { "n_slots": 3 }, "parent": "r:108", "results": [ "s:153" ] }, { "annotation": null, "id": "r:142", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:154" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:141", "results": [ "s:153" ] }, { "annotation": null, "id": "r:143", "op": "linear.add_const", "operands": [ "s:120" ], "params": { "scalar": -1.0 }, "parent": "r:108", "results": [ "s:155" ] }, { "annotation": null, "id": "r:144", "op": "linear.add_const", "operands": [ "s:155" ], "params": { "scalar": 1.0 }, "parent": "r:108", "results": [ "s:156" ] }, { "annotation": null, "id": "r:145", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": "r:108", "results": [ "s:157" ] }, { "annotation": null, "id": "r:146", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:129", "s:160" ], "params": {}, "parent": "r:108", "results": [ "s:158" ] }, { "annotation": null, "id": "r:147", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:159" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 5 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 4 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 5 ] } } }, "parent": "r:146", "results": [ "s:158" ] }, { "annotation": null, "id": "r:148", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 4 ] } } }, "parent": "r:108", "results": [ "s:160" ] }, { "annotation": null, "id": "r:149", "op": "swiglu.map_select.select", "operands": [ "s:140", "s:8", "s:163" ], "params": {}, "parent": "r:108", "results": [ "s:161" ] }, { "annotation": null, "id": "r:150", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:162" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:149", "results": [ "s:161" ] }, { "annotation": null, "id": "r:151", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:108", "results": [ "s:163" ] }, { "annotation": null, "id": "r:152", "op": "attention_ops.get_prev_value", "operands": [ "s:170", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:164" ] }, { "annotation": null, "id": "r:153", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:176" ], "params": {}, "parent": null, "results": [ "s:170" ] }, { "annotation": null, "id": "r:154", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:171", "s:199", "s:217" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:153", "results": [ "s:170" ] }, { "annotation": null, "id": "r:155", "op": "linear.concat", "operands": [ "s:172", "s:197" ], "params": {}, "parent": "r:153", "results": [ "s:171" ] }, { "annotation": null, "id": "r:156", "op": "linear.bool_to_01", "operands": [ "s:174" ], "params": {}, "parent": "r:153", "results": [ "s:172" ] }, { "annotation": null, "id": "r:157", "op": "linear.multiply_const", "operands": [ "s:173" ], "params": { "scalar": 0.5 }, "parent": "r:156", "results": [ "s:172" ] }, { "annotation": null, "id": "r:158", "op": "linear.add_const", "operands": [ "s:174" ], "params": { "scalar": 1.0 }, "parent": "r:156", "results": [ "s:173" ] }, { "annotation": null, "id": "r:159", "op": "swiglu.map_select.in_range", "operands": [ "s:176", "s:196" ], "params": { "n_slots": 3 }, "parent": "r:153", "results": [ "s:174" ] }, { "annotation": null, "id": "r:160", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:175" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:159", "results": [ "s:174" ] }, { "annotation": null, "id": "r:161", "op": "linear.add_const", "operands": [ "s:177" ], "params": { "scalar": -3.0 }, "parent": null, "results": [ "s:176" ] }, { "annotation": null, "id": "r:162", "op": "swiglu.sequence_ops.IndexedRegion.length_at", "operands": [ "s:7" ], "params": {}, "parent": null, "results": [ "s:177" ] }, { "annotation": null, "id": "r:163", "op": "linear.add_const", "operands": [ "s:178" ], "params": { "scalar": -1.0 }, "parent": "r:162", "results": [ "s:177" ] }, { "annotation": null, "id": "r:164", "op": "attention_ops.get_prev_value", "operands": [ "s:184", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:162", "results": [ "s:178" ] }, { "annotation": null, "id": "r:165", "op": "swiglu.sequence_ops.IndexedRegion.__init__", "operands": [ "s:8", "s:51", "s:203" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "max_len": 3, "max_read_distance": 10, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [] }, { "annotation": null, "id": "r:166", "op": "swiglu.marker_count.count_since_marker", "operands": [ "s:188", "s:194" ], "params": { "max_gap": 4, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:165", "results": [ "s:184" ] }, { "annotation": null, "id": "r:167", "op": "linear.add_const", "operands": [ "s:185" ], "params": { "scalar": -1.0 }, "parent": "r:166", "results": [ "s:184" ] }, { "annotation": null, "id": "r:168", "op": "swiglu.arithmetic_ops.reciprocal", "operands": [ "s:186" ], "params": { "max_value": 1.5, "min_value": 0.18181818181818182, "step": 0.04252199413489736 }, "parent": "r:166", "results": [ "s:185" ] }, { "annotation": null, "id": "r:169", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:186" ], "params": { "breakpoints": [ 0.18181818181818182, 0.19462576879929028, 0.20833554434393137, 0.2230110601738595, 0.23872034470395204, 0.2555362183882104, 0.2735366312792783, 0.2928050243662364, 0.3134307163656553, 0.33550931775887877, 0.35914317399480933, 0.3844418399126637, 0.4115225875838872, 0.44051094992733025, 0.4715413026176177, 0.5047574869841495, 0.5403134767881832, 0.5783740919688469, 0.6191157626666555, 0.6627273470661621, 0.7094110068488624, 0.7593831443145113, 0.8128754055148878, 0.8701357540500343, 0.9314296205045596, 0.997041132852221, 1.0672744335323334, 1.142455089303316, 1.2229316004087583, 1.3090770160517557, 1.4012906636660458, 1.5 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 20.739660046174524, "name": "reciprocal" }, "parent": "r:168", "results": [ "s:185" ] }, { "annotation": null, "id": "r:170", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:186" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 32 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "name": "reciprocal_0_32", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:169", "results": [ "s:185" ] }, { "annotation": null, "id": "r:171", "op": "attention_ops.attend_mean_where", "operands": [ "s:188", "s:194" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:166", "results": [ "s:186" ] }, { "annotation": null, "id": "r:172", "op": "attention_ops.get_prev_value", "operands": [ "s:51", "s:51" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:165", "results": [ "s:188" ] }, { "annotation": null, "id": "r:173", "op": "linear.bool_to_01", "operands": [ "s:51" ], "params": {}, "parent": "r:165", "results": [ "s:194" ] }, { "annotation": null, "id": "r:174", "op": "linear.multiply_const", "operands": [ "s:195" ], "params": { "scalar": 0.5 }, "parent": "r:173", "results": [ "s:194" ] }, { "annotation": null, "id": "r:175", "op": "linear.add_const", "operands": [ "s:51" ], "params": { "scalar": 1.0 }, "parent": "r:173", "results": [ "s:195" ] }, { "annotation": null, "id": "r:176", "op": "linear.add_const", "operands": [ "s:176" ], "params": { "scalar": 1.0 }, "parent": "r:153", "results": [ "s:196" ] }, { "annotation": null, "id": "r:177", "op": "linear.add_const", "operands": [ "s:198" ], "params": { "scalar": 1.0 }, "parent": "r:153", "results": [ "s:197" ] }, { "annotation": null, "id": "r:178", "op": "linear.add", "operands": [ "s:200", "s:214" ], "params": {}, "parent": "r:165", "results": [ "s:199" ] }, { "annotation": null, "id": "r:179", "op": "linear.concat", "operands": [ "s:201", "s:213" ], "params": {}, "parent": "r:165", "results": [ "s:200" ] }, { "annotation": null, "id": "r:180", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:203", "s:207" ], "params": {}, "parent": "r:165", "results": [ "s:201" ] }, { "annotation": null, "id": "r:181", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:202" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 4 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 3 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 4 ] } } }, "parent": "r:180", "results": [ "s:201" ] }, { "annotation": null, "id": "r:182", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:143", "s:145", "s:206" ], "params": {}, "parent": null, "results": [ "s:203" ] }, { "annotation": null, "id": "r:183", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:204" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 2.0, "true_level": 1.0 }, "parent": "r:182", "results": [ "s:203" ] }, { "annotation": null, "id": "r:184", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:204" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:183", "results": [ "s:203" ] }, { "annotation": null, "id": "r:185", "op": "linear.sum_nodes", "operands": [ "s:143", "s:145", "s:206" ], "params": { "max_fanout": null }, "parent": "r:182", "results": [ "s:204" ] }, { "annotation": null, "id": "r:186", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:64" ], "params": {}, "parent": null, "results": [ "s:206" ] }, { "annotation": null, "id": "r:187", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:64" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:186", "results": [ "s:206" ] }, { "annotation": null, "id": "r:188", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:64" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:187", "results": [ "s:206" ] }, { "annotation": null, "id": "r:189", "op": "linear.bool_to_01", "operands": [ "s:209" ], "params": {}, "parent": "r:165", "results": [ "s:207" ] }, { "annotation": null, "id": "r:190", "op": "linear.multiply_const", "operands": [ "s:208" ], "params": { "scalar": 0.5 }, "parent": "r:189", "results": [ "s:207" ] }, { "annotation": null, "id": "r:191", "op": "linear.add_const", "operands": [ "s:209" ], "params": { "scalar": 1.0 }, "parent": "r:189", "results": [ "s:208" ] }, { "annotation": null, "id": "r:192", "op": "swiglu.map_select.in_range", "operands": [ "s:211", "s:212" ], "params": { "n_slots": 3 }, "parent": "r:165", "results": [ "s:209" ] }, { "annotation": null, "id": "r:193", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:210" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:192", "results": [ "s:209" ] }, { "annotation": null, "id": "r:194", "op": "linear.add_const", "operands": [ "s:184" ], "params": { "scalar": -1.0 }, "parent": "r:165", "results": [ "s:211" ] }, { "annotation": null, "id": "r:195", "op": "linear.add_const", "operands": [ "s:211" ], "params": { "scalar": 1.0 }, "parent": "r:165", "results": [ "s:212" ] }, { "annotation": null, "id": "r:196", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": "r:165", "results": [ "s:213" ] }, { "annotation": null, "id": "r:197", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:51", "s:216" ], "params": {}, "parent": "r:165", "results": [ "s:214" ] }, { "annotation": null, "id": "r:198", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:215" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 5 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 4 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 5 ] } } }, "parent": "r:197", "results": [ "s:214" ] }, { "annotation": null, "id": "r:199", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 4 ] } } }, "parent": "r:165", "results": [ "s:216" ] }, { "annotation": null, "id": "r:200", "op": "swiglu.map_select.select", "operands": [ "s:203", "s:8", "s:219" ], "params": {}, "parent": "r:165", "results": [ "s:217" ] }, { "annotation": null, "id": "r:201", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:218" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:200", "results": [ "s:217" ] }, { "annotation": null, "id": "r:202", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:165", "results": [ "s:219" ] }, { "annotation": null, "id": "r:203", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:221" ], "params": { "breakpoints": [ 9.5, 9.6, 19.5, 19.6, 29.5, 29.6, 39.5, 39.6, 49.5, 49.6, 59.5, 59.6 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 10.0, "name": "carry_staircase" }, "parent": null, "results": [ "s:220" ] }, { "annotation": null, "id": "r:204", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:221" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 1 ] } }, "name": "carry_staircase_0_12", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:203", "results": [ "s:220" ] }, { "annotation": null, "id": "r:205", "op": "linear.add", "operands": [ "s:222", "s:263" ], "params": {}, "parent": null, "results": [ "s:221" ] }, { "annotation": null, "id": "r:206", "op": "linear.sum_nodes", "operands": [ "s:224", "s:225", "s:244" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:222" ] }, { "annotation": null, "id": "r:207", "op": "linear.slice_columns", "operands": [ "s:98" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:224" ] }, { "annotation": null, "id": "r:208", "op": "linear.slice_columns", "operands": [ "s:226" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:225" ] }, { "annotation": null, "id": "r:209", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:227" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:226" ] }, { "annotation": null, "id": "r:210", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:227" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:209", "results": [ "s:226" ] }, { "annotation": null, "id": "r:211", "op": "linear.concat", "operands": [ "s:100", "s:228" ], "params": {}, "parent": null, "results": [ "s:227" ] }, { "annotation": null, "id": "r:212", "op": "attention_ops.get_prev_value", "operands": [ "s:234", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:228" ] }, { "annotation": null, "id": "r:213", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:240" ], "params": {}, "parent": null, "results": [ "s:234" ] }, { "annotation": null, "id": "r:214", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:235", "s:199", "s:217" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:213", "results": [ "s:234" ] }, { "annotation": null, "id": "r:215", "op": "linear.concat", "operands": [ "s:236", "s:242" ], "params": {}, "parent": "r:213", "results": [ "s:235" ] }, { "annotation": null, "id": "r:216", "op": "linear.bool_to_01", "operands": [ "s:238" ], "params": {}, "parent": "r:213", "results": [ "s:236" ] }, { "annotation": null, "id": "r:217", "op": "linear.multiply_const", "operands": [ "s:237" ], "params": { "scalar": 0.5 }, "parent": "r:216", "results": [ "s:236" ] }, { "annotation": null, "id": "r:218", "op": "linear.add_const", "operands": [ "s:238" ], "params": { "scalar": 1.0 }, "parent": "r:216", "results": [ "s:237" ] }, { "annotation": null, "id": "r:219", "op": "swiglu.map_select.in_range", "operands": [ "s:240", "s:241" ], "params": { "n_slots": 3 }, "parent": "r:213", "results": [ "s:238" ] }, { "annotation": null, "id": "r:220", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:239" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:219", "results": [ "s:238" ] }, { "annotation": null, "id": "r:221", "op": "linear.add_const", "operands": [ "s:177" ], "params": { "scalar": -2.0 }, "parent": null, "results": [ "s:240" ] }, { "annotation": null, "id": "r:222", "op": "linear.add_const", "operands": [ "s:240" ], "params": { "scalar": 1.0 }, "parent": "r:213", "results": [ "s:241" ] }, { "annotation": null, "id": "r:223", "op": "linear.add_const", "operands": [ "s:243" ], "params": { "scalar": 1.0 }, "parent": "r:213", "results": [ "s:242" ] }, { "annotation": null, "id": "r:224", "op": "linear.slice_columns", "operands": [ "s:245" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:244" ] }, { "annotation": null, "id": "r:225", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:246" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:245" ] }, { "annotation": null, "id": "r:226", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:246" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:225", "results": [ "s:245" ] }, { "annotation": null, "id": "r:227", "op": "linear.concat", "operands": [ "s:247", "s:164" ], "params": {}, "parent": null, "results": [ "s:246" ] }, { "annotation": null, "id": "r:228", "op": "attention_ops.get_prev_value", "operands": [ "s:253", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:247" ] }, { "annotation": null, "id": "r:229", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:259" ], "params": {}, "parent": null, "results": [ "s:253" ] }, { "annotation": null, "id": "r:230", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:254", "s:136", "s:161" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:229", "results": [ "s:253" ] }, { "annotation": null, "id": "r:231", "op": "linear.concat", "operands": [ "s:255", "s:261" ], "params": {}, "parent": "r:229", "results": [ "s:254" ] }, { "annotation": null, "id": "r:232", "op": "linear.bool_to_01", "operands": [ "s:257" ], "params": {}, "parent": "r:229", "results": [ "s:255" ] }, { "annotation": null, "id": "r:233", "op": "linear.multiply_const", "operands": [ "s:256" ], "params": { "scalar": 0.5 }, "parent": "r:232", "results": [ "s:255" ] }, { "annotation": null, "id": "r:234", "op": "linear.add_const", "operands": [ "s:257" ], "params": { "scalar": 1.0 }, "parent": "r:232", "results": [ "s:256" ] }, { "annotation": null, "id": "r:235", "op": "swiglu.map_select.in_range", "operands": [ "s:259", "s:260" ], "params": { "n_slots": 3 }, "parent": "r:229", "results": [ "s:257" ] }, { "annotation": null, "id": "r:236", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:258" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:235", "results": [ "s:257" ] }, { "annotation": null, "id": "r:237", "op": "linear.add_const", "operands": [ "s:113" ], "params": { "scalar": -2.0 }, "parent": null, "results": [ "s:259" ] }, { "annotation": null, "id": "r:238", "op": "linear.add_const", "operands": [ "s:259" ], "params": { "scalar": 1.0 }, "parent": "r:229", "results": [ "s:260" ] }, { "annotation": null, "id": "r:239", "op": "linear.add_const", "operands": [ "s:262" ], "params": { "scalar": 1.0 }, "parent": "r:229", "results": [ "s:261" ] }, { "annotation": null, "id": "r:240", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:264" ], "params": { "breakpoints": [ 9.5, 9.6, 19.5, 19.6, 29.5, 29.6, 39.5, 39.6, 49.5, 49.6, 59.5, 59.6 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 10.0, "name": "carry_staircase" }, "parent": null, "results": [ "s:263" ] }, { "annotation": null, "id": "r:241", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:264" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 1 ] } }, "name": "carry_staircase_0_12", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:240", "results": [ "s:263" ] }, { "annotation": null, "id": "r:242", "op": "linear.add", "operands": [ "s:265", "s:310" ], "params": {}, "parent": null, "results": [ "s:264" ] }, { "annotation": null, "id": "r:243", "op": "linear.sum_nodes", "operands": [ "s:267", "s:268", "s:287", "s:288", "s:291" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:265" ] }, { "annotation": null, "id": "r:244", "op": "linear.slice_columns", "operands": [ "s:226" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:267" ] }, { "annotation": null, "id": "r:245", "op": "linear.slice_columns", "operands": [ "s:269" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:268" ] }, { "annotation": null, "id": "r:246", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:270" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:269" ] }, { "annotation": null, "id": "r:247", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:270" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:246", "results": [ "s:269" ] }, { "annotation": null, "id": "r:248", "op": "linear.concat", "operands": [ "s:100", "s:271" ], "params": {}, "parent": null, "results": [ "s:270" ] }, { "annotation": null, "id": "r:249", "op": "attention_ops.get_prev_value", "operands": [ "s:277", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:271" ] }, { "annotation": null, "id": "r:250", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:283" ], "params": {}, "parent": null, "results": [ "s:277" ] }, { "annotation": null, "id": "r:251", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:278", "s:199", "s:217" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:250", "results": [ "s:277" ] }, { "annotation": null, "id": "r:252", "op": "linear.concat", "operands": [ "s:279", "s:285" ], "params": {}, "parent": "r:250", "results": [ "s:278" ] }, { "annotation": null, "id": "r:253", "op": "linear.bool_to_01", "operands": [ "s:281" ], "params": {}, "parent": "r:250", "results": [ "s:279" ] }, { "annotation": null, "id": "r:254", "op": "linear.multiply_const", "operands": [ "s:280" ], "params": { "scalar": 0.5 }, "parent": "r:253", "results": [ "s:279" ] }, { "annotation": null, "id": "r:255", "op": "linear.add_const", "operands": [ "s:281" ], "params": { "scalar": 1.0 }, "parent": "r:253", "results": [ "s:280" ] }, { "annotation": null, "id": "r:256", "op": "swiglu.map_select.in_range", "operands": [ "s:283", "s:284" ], "params": { "n_slots": 3 }, "parent": "r:250", "results": [ "s:281" ] }, { "annotation": null, "id": "r:257", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:282" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:256", "results": [ "s:281" ] }, { "annotation": null, "id": "r:258", "op": "linear.add_const", "operands": [ "s:177" ], "params": { "scalar": -1.0 }, "parent": null, "results": [ "s:283" ] }, { "annotation": null, "id": "r:259", "op": "linear.add_const", "operands": [ "s:283" ], "params": { "scalar": 1.0 }, "parent": "r:250", "results": [ "s:284" ] }, { "annotation": null, "id": "r:260", "op": "linear.add_const", "operands": [ "s:286" ], "params": { "scalar": 1.0 }, "parent": "r:250", "results": [ "s:285" ] }, { "annotation": null, "id": "r:261", "op": "linear.slice_columns", "operands": [ "s:245" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:287" ] }, { "annotation": null, "id": "r:262", "op": "linear.slice_columns", "operands": [ "s:289" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:288" ] }, { "annotation": null, "id": "r:263", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:290" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:289" ] }, { "annotation": null, "id": "r:264", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:290" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:263", "results": [ "s:289" ] }, { "annotation": null, "id": "r:265", "op": "linear.concat", "operands": [ "s:247", "s:228" ], "params": {}, "parent": null, "results": [ "s:290" ] }, { "annotation": null, "id": "r:266", "op": "linear.slice_columns", "operands": [ "s:292" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:291" ] }, { "annotation": null, "id": "r:267", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:293" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:292" ] }, { "annotation": null, "id": "r:268", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:293" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:267", "results": [ "s:292" ] }, { "annotation": null, "id": "r:269", "op": "linear.concat", "operands": [ "s:294", "s:164" ], "params": {}, "parent": null, "results": [ "s:293" ] }, { "annotation": null, "id": "r:270", "op": "attention_ops.get_prev_value", "operands": [ "s:300", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:294" ] }, { "annotation": null, "id": "r:271", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:306" ], "params": {}, "parent": null, "results": [ "s:300" ] }, { "annotation": null, "id": "r:272", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:301", "s:136", "s:161" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:271", "results": [ "s:300" ] }, { "annotation": null, "id": "r:273", "op": "linear.concat", "operands": [ "s:302", "s:308" ], "params": {}, "parent": "r:271", "results": [ "s:301" ] }, { "annotation": null, "id": "r:274", "op": "linear.bool_to_01", "operands": [ "s:304" ], "params": {}, "parent": "r:271", "results": [ "s:302" ] }, { "annotation": null, "id": "r:275", "op": "linear.multiply_const", "operands": [ "s:303" ], "params": { "scalar": 0.5 }, "parent": "r:274", "results": [ "s:302" ] }, { "annotation": null, "id": "r:276", "op": "linear.add_const", "operands": [ "s:304" ], "params": { "scalar": 1.0 }, "parent": "r:274", "results": [ "s:303" ] }, { "annotation": null, "id": "r:277", "op": "swiglu.map_select.in_range", "operands": [ "s:306", "s:307" ], "params": { "n_slots": 3 }, "parent": "r:271", "results": [ "s:304" ] }, { "annotation": null, "id": "r:278", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:305" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:277", "results": [ "s:304" ] }, { "annotation": null, "id": "r:279", "op": "linear.add_const", "operands": [ "s:113" ], "params": { "scalar": -1.0 }, "parent": null, "results": [ "s:306" ] }, { "annotation": null, "id": "r:280", "op": "linear.add_const", "operands": [ "s:306" ], "params": { "scalar": 1.0 }, "parent": "r:271", "results": [ "s:307" ] }, { "annotation": null, "id": "r:281", "op": "linear.add_const", "operands": [ "s:309" ], "params": { "scalar": 1.0 }, "parent": "r:271", "results": [ "s:308" ] }, { "annotation": null, "id": "r:282", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:311" ], "params": { "breakpoints": [ 9.5, 9.6, 19.5, 19.6, 29.5, 29.6, 39.5, 39.6, 49.5, 49.6, 59.5, 59.6 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 10.0, "name": "carry_staircase" }, "parent": null, "results": [ "s:310" ] }, { "annotation": null, "id": "r:283", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:311" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 1 ] } }, "name": "carry_staircase_0_12", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:282", "results": [ "s:310" ] }, { "annotation": null, "id": "r:284", "op": "linear.add", "operands": [ "s:312", "s:323" ], "params": {}, "parent": null, "results": [ "s:311" ] }, { "annotation": null, "id": "r:285", "op": "linear.sum_nodes", "operands": [ "s:314", "s:315", "s:316", "s:319", "s:320" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:312" ] }, { "annotation": null, "id": "r:286", "op": "linear.slice_columns", "operands": [ "s:269" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:314" ] }, { "annotation": null, "id": "r:287", "op": "linear.slice_columns", "operands": [ "s:289" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:315" ] }, { "annotation": null, "id": "r:288", "op": "linear.slice_columns", "operands": [ "s:317" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:316" ] }, { "annotation": null, "id": "r:289", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:318" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:317" ] }, { "annotation": null, "id": "r:290", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:318" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:289", "results": [ "s:317" ] }, { "annotation": null, "id": "r:291", "op": "linear.concat", "operands": [ "s:247", "s:271" ], "params": {}, "parent": null, "results": [ "s:318" ] }, { "annotation": null, "id": "r:292", "op": "linear.slice_columns", "operands": [ "s:292" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:319" ] }, { "annotation": null, "id": "r:293", "op": "linear.slice_columns", "operands": [ "s:321" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:320" ] }, { "annotation": null, "id": "r:294", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:322" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:321" ] }, { "annotation": null, "id": "r:295", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:322" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:294", "results": [ "s:321" ] }, { "annotation": null, "id": "r:296", "op": "linear.concat", "operands": [ "s:294", "s:228" ], "params": {}, "parent": null, "results": [ "s:322" ] }, { "annotation": null, "id": "r:297", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:324" ], "params": { "breakpoints": [ 9.5, 9.6, 19.5, 19.6, 29.5, 29.6, 39.5, 39.6, 49.5, 49.6, 59.5, 59.6 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 10.0, "name": "carry_staircase" }, "parent": null, "results": [ "s:323" ] }, { "annotation": null, "id": "r:298", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:324" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 1 ] } }, "name": "carry_staircase_0_12", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:297", "results": [ "s:323" ] }, { "annotation": null, "id": "r:299", "op": "linear.add", "operands": [ "s:325", "s:332" ], "params": {}, "parent": null, "results": [ "s:324" ] }, { "annotation": null, "id": "r:300", "op": "linear.sum_nodes", "operands": [ "s:327", "s:328", "s:329" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:325" ] }, { "annotation": null, "id": "r:301", "op": "linear.slice_columns", "operands": [ "s:317" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:327" ] }, { "annotation": null, "id": "r:302", "op": "linear.slice_columns", "operands": [ "s:321" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:328" ] }, { "annotation": null, "id": "r:303", "op": "linear.slice_columns", "operands": [ "s:330" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:329" ] }, { "annotation": null, "id": "r:304", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:331" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:330" ] }, { "annotation": null, "id": "r:305", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:331" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:304", "results": [ "s:330" ] }, { "annotation": null, "id": "r:306", "op": "linear.concat", "operands": [ "s:294", "s:271" ], "params": {}, "parent": null, "results": [ "s:331" ] }, { "annotation": null, "id": "r:307", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:333" ], "params": { "breakpoints": [ 9.5, 9.6, 19.5, 19.6, 29.5, 29.6, 39.5, 39.6, 49.5, 49.6, 59.5, 59.6 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 10.0, "name": "carry_staircase" }, "parent": null, "results": [ "s:332" ] }, { "annotation": null, "id": "r:308", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:333" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 1 ] } }, "name": "carry_staircase_0_12", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:307", "results": [ "s:332" ] }, { "annotation": null, "id": "r:309", "op": "linear.add", "operands": [ "s:334", "s:337" ], "params": {}, "parent": null, "results": [ "s:333" ] }, { "annotation": null, "id": "r:310", "op": "linear.sum_nodes", "operands": [ "s:336" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:334" ] }, { "annotation": null, "id": "r:311", "op": "linear.slice_columns", "operands": [ "s:330" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:336" ] }, { "annotation": null, "id": "r:312", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:337" ] }, { "annotation": null, "id": "r:313", "op": "linear.add_const", "operands": [ "s:94" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:338" ] }, { "annotation": null, "id": "r:314", "op": "linear.bool_to_01", "operands": [ "s:341" ], "params": {}, "parent": "r:28", "results": [ "s:339" ] }, { "annotation": null, "id": "r:315", "op": "linear.multiply_const", "operands": [ "s:340" ], "params": { "scalar": 0.5 }, "parent": "r:314", "results": [ "s:339" ] }, { "annotation": null, "id": "r:316", "op": "linear.add_const", "operands": [ "s:341" ], "params": { "scalar": 1.0 }, "parent": "r:314", "results": [ "s:340" ] }, { "annotation": null, "id": "r:317", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:342" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.5, "true_level": 1.0 }, "parent": "r:28", "results": [ "s:341" ] }, { "annotation": null, "id": "r:318", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:342" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:317", "results": [ "s:341" ] }, { "annotation": null, "id": "r:319", "op": "linear.sum_nodes", "operands": [ "s:39", "s:344" ], "params": { "max_fanout": null }, "parent": "r:28", "results": [ "s:342" ] }, { "annotation": null, "id": "r:320", "op": "linear.bool_to_01", "operands": [ "s:346" ], "params": {}, "parent": "r:28", "results": [ "s:344" ] }, { "annotation": null, "id": "r:321", "op": "linear.multiply_const", "operands": [ "s:345" ], "params": { "scalar": 0.5 }, "parent": "r:320", "results": [ "s:344" ] }, { "annotation": null, "id": "r:322", "op": "linear.add_const", "operands": [ "s:346" ], "params": { "scalar": 1.0 }, "parent": "r:320", "results": [ "s:345" ] }, { "annotation": null, "id": "r:323", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:347" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:28", "results": [ "s:346" ] }, { "annotation": null, "id": "r:324", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:347" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:323", "results": [ "s:346" ] }, { "annotation": null, "id": "r:325", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:71", "s:80", "s:355" ], "params": {}, "parent": null, "results": [ "s:347" ] }, { "annotation": null, "id": "r:326", "op": "linear.sum_nodes", "operands": [ "s:349", "s:351", "s:353" ], "params": { "max_fanout": null }, "parent": "r:325", "results": [ "s:347" ] }, { "annotation": null, "id": "r:327", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:71" ], "params": {}, "parent": "r:325", "results": [ "s:349" ] }, { "annotation": null, "id": "r:328", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:350" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:327", "results": [ "s:349" ] }, { "annotation": null, "id": "r:329", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:80" ], "params": {}, "parent": "r:325", "results": [ "s:351" ] }, { "annotation": null, "id": "r:330", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:352" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:329", "results": [ "s:351" ] }, { "annotation": null, "id": "r:331", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:355" ], "params": {}, "parent": "r:325", "results": [ "s:353" ] }, { "annotation": null, "id": "r:332", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:354" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:331", "results": [ "s:353" ] }, { "annotation": null, "id": "r:333", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:356" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:355" ] }, { "annotation": null, "id": "r:334", "op": "linear.bool_to_01", "operands": [ "s:358" ], "params": {}, "parent": null, "results": [ "s:356" ] }, { "annotation": null, "id": "r:335", "op": "linear.multiply_const", "operands": [ "s:357" ], "params": { "scalar": 0.5 }, "parent": "r:334", "results": [ "s:356" ] }, { "annotation": null, "id": "r:336", "op": "linear.add_const", "operands": [ "s:358" ], "params": { "scalar": 1.0 }, "parent": "r:334", "results": [ "s:357" ] }, { "annotation": null, "id": "r:337", "op": "swiglu.map_select.in_range", "operands": [ "s:221", "s:360" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:358" ] }, { "annotation": null, "id": "r:338", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:359" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:337", "results": [ "s:358" ] }, { "annotation": null, "id": "r:339", "op": "linear.add_const", "operands": [ "s:221" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:360" ] }, { "annotation": null, "id": "r:340", "op": "linear.bool_to_01", "operands": [ "s:363" ], "params": {}, "parent": "r:28", "results": [ "s:361" ] }, { "annotation": null, "id": "r:341", "op": "linear.multiply_const", "operands": [ "s:362" ], "params": { "scalar": 0.5 }, "parent": "r:340", "results": [ "s:361" ] }, { "annotation": null, "id": "r:342", "op": "linear.add_const", "operands": [ "s:363" ], "params": { "scalar": 1.0 }, "parent": "r:340", "results": [ "s:362" ] }, { "annotation": null, "id": "r:343", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:364" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 2.5, "true_level": 1.0 }, "parent": "r:28", "results": [ "s:363" ] }, { "annotation": null, "id": "r:344", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:364" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:343", "results": [ "s:363" ] }, { "annotation": null, "id": "r:345", "op": "linear.sum_nodes", "operands": [ "s:39", "s:344", "s:366" ], "params": { "max_fanout": null }, "parent": "r:28", "results": [ "s:364" ] }, { "annotation": null, "id": "r:346", "op": "linear.bool_to_01", "operands": [ "s:368" ], "params": {}, "parent": "r:28", "results": [ "s:366" ] }, { "annotation": null, "id": "r:347", "op": "linear.multiply_const", "operands": [ "s:367" ], "params": { "scalar": 0.5 }, "parent": "r:346", "results": [ "s:366" ] }, { "annotation": null, "id": "r:348", "op": "linear.add_const", "operands": [ "s:368" ], "params": { "scalar": 1.0 }, "parent": "r:346", "results": [ "s:367" ] }, { "annotation": null, "id": "r:349", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:369" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:28", "results": [ "s:368" ] }, { "annotation": null, "id": "r:350", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:369" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:349", "results": [ "s:368" ] }, { "annotation": null, "id": "r:351", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:373", "s:80", "s:387" ], "params": {}, "parent": null, "results": [ "s:369" ] }, { "annotation": null, "id": "r:352", "op": "linear.sum_nodes", "operands": [ "s:371", "s:383", "s:385" ], "params": { "max_fanout": null }, "parent": "r:351", "results": [ "s:369" ] }, { "annotation": null, "id": "r:353", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:373" ], "params": {}, "parent": "r:351", "results": [ "s:371" ] }, { "annotation": null, "id": "r:354", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:372" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:353", "results": [ "s:371" ] }, { "annotation": null, "id": "r:355", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:374" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:373" ] }, { "annotation": null, "id": "r:356", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:374" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:355", "results": [ "s:373" ] }, { "annotation": null, "id": "r:357", "op": "linear.concat", "operands": [ "s:375", "s:375", "s:376" ], "params": {}, "parent": null, "results": [ "s:374" ] }, { "annotation": null, "id": "r:358", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:375" ] }, { "annotation": null, "id": "r:359", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:377" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:376" ] }, { "annotation": null, "id": "r:360", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:377" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:359", "results": [ "s:376" ] }, { "annotation": null, "id": "r:361", "op": "linear.concat", "operands": [ "s:100", "s:164", "s:378" ], "params": {}, "parent": null, "results": [ "s:377" ] }, { "annotation": null, "id": "r:362", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:379" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:378" ] }, { "annotation": null, "id": "r:363", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:379" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:362", "results": [ "s:378" ] }, { "annotation": null, "id": "r:364", "op": "linear.concat", "operands": [ "s:247", "s:228", "s:380" ], "params": {}, "parent": null, "results": [ "s:379" ] }, { "annotation": null, "id": "r:365", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:381" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:380" ] }, { "annotation": null, "id": "r:366", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:381" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:365", "results": [ "s:380" ] }, { "annotation": null, "id": "r:367", "op": "linear.concat", "operands": [ "s:294", "s:271", "s:382" ], "params": {}, "parent": null, "results": [ "s:381" ] }, { "annotation": null, "id": "r:368", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 2 ] } } }, "parent": null, "results": [ "s:382" ] }, { "annotation": null, "id": "r:369", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:80" ], "params": {}, "parent": "r:351", "results": [ "s:383" ] }, { "annotation": null, "id": "r:370", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:384" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:369", "results": [ "s:383" ] }, { "annotation": null, "id": "r:371", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:387" ], "params": {}, "parent": "r:351", "results": [ "s:385" ] }, { "annotation": null, "id": "r:372", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:386" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:371", "results": [ "s:385" ] }, { "annotation": null, "id": "r:373", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:388" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:387" ] }, { "annotation": null, "id": "r:374", "op": "linear.bool_to_01", "operands": [ "s:390" ], "params": {}, "parent": null, "results": [ "s:388" ] }, { "annotation": null, "id": "r:375", "op": "linear.multiply_const", "operands": [ "s:389" ], "params": { "scalar": 0.5 }, "parent": "r:374", "results": [ "s:388" ] }, { "annotation": null, "id": "r:376", "op": "linear.add_const", "operands": [ "s:390" ], "params": { "scalar": 1.0 }, "parent": "r:374", "results": [ "s:389" ] }, { "annotation": null, "id": "r:377", "op": "swiglu.map_select.in_range", "operands": [ "s:264", "s:392" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:390" ] }, { "annotation": null, "id": "r:378", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:391" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:377", "results": [ "s:390" ] }, { "annotation": null, "id": "r:379", "op": "linear.add_const", "operands": [ "s:264" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:392" ] }, { "annotation": null, "id": "r:380", "op": "linear.bool_to_01", "operands": [ "s:395" ], "params": {}, "parent": "r:28", "results": [ "s:393" ] }, { "annotation": null, "id": "r:381", "op": "linear.multiply_const", "operands": [ "s:394" ], "params": { "scalar": 0.5 }, "parent": "r:380", "results": [ "s:393" ] }, { "annotation": null, "id": "r:382", "op": "linear.add_const", "operands": [ "s:395" ], "params": { "scalar": 1.0 }, "parent": "r:380", "results": [ "s:394" ] }, { "annotation": null, "id": "r:383", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:396" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 3.5, "true_level": 1.0 }, "parent": "r:28", "results": [ "s:395" ] }, { "annotation": null, "id": "r:384", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:396" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:383", "results": [ "s:395" ] }, { "annotation": null, "id": "r:385", "op": "linear.sum_nodes", "operands": [ "s:39", "s:344", "s:366", "s:398" ], "params": { "max_fanout": null }, "parent": "r:28", "results": [ "s:396" ] }, { "annotation": null, "id": "r:386", "op": "linear.bool_to_01", "operands": [ "s:400" ], "params": {}, "parent": "r:28", "results": [ "s:398" ] }, { "annotation": null, "id": "r:387", "op": "linear.multiply_const", "operands": [ "s:399" ], "params": { "scalar": 0.5 }, "parent": "r:386", "results": [ "s:398" ] }, { "annotation": null, "id": "r:388", "op": "linear.add_const", "operands": [ "s:400" ], "params": { "scalar": 1.0 }, "parent": "r:386", "results": [ "s:399" ] }, { "annotation": null, "id": "r:389", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:401" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:28", "results": [ "s:400" ] }, { "annotation": null, "id": "r:390", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:401" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:389", "results": [ "s:400" ] }, { "annotation": null, "id": "r:391", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:405", "s:408", "s:444" ], "params": {}, "parent": null, "results": [ "s:401" ] }, { "annotation": null, "id": "r:392", "op": "linear.sum_nodes", "operands": [ "s:403", "s:406", "s:442" ], "params": { "max_fanout": null }, "parent": "r:391", "results": [ "s:401" ] }, { "annotation": null, "id": "r:393", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:405" ], "params": {}, "parent": "r:391", "results": [ "s:403" ] }, { "annotation": null, "id": "r:394", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:404" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:393", "results": [ "s:403" ] }, { "annotation": null, "id": "r:395", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:377" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:405" ] }, { "annotation": null, "id": "r:396", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:377" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:395", "results": [ "s:405" ] }, { "annotation": null, "id": "r:397", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:408" ], "params": {}, "parent": "r:391", "results": [ "s:406" ] }, { "annotation": null, "id": "r:398", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:407" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:397", "results": [ "s:406" ] }, { "annotation": null, "id": "r:399", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:409" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:408" ] }, { "annotation": null, "id": "r:400", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:409" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:399", "results": [ "s:408" ] }, { "annotation": null, "id": "r:401", "op": "linear.concat", "operands": [ "s:410", "s:427", "s:429" ], "params": {}, "parent": null, "results": [ "s:409" ] }, { "annotation": null, "id": "r:402", "op": "swiglu.map_select.select", "operands": [ "s:412", "s:100", "s:164" ], "params": {}, "parent": null, "results": [ "s:410" ] }, { "annotation": null, "id": "r:403", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:411" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:402", "results": [ "s:410" ] }, { "annotation": null, "id": "r:404", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:413" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:412" ] }, { "annotation": null, "id": "r:405", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:413" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:404", "results": [ "s:412" ] }, { "annotation": null, "id": "r:406", "op": "linear.concat", "operands": [ "s:415", "s:418", "s:419", "s:422", "s:423", "s:426" ], "params": {}, "parent": null, "results": [ "s:414" ] }, { "annotation": null, "id": "r:407", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:416" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:415" ] }, { "annotation": null, "id": "r:408", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:416" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:407", "results": [ "s:415" ] }, { "annotation": null, "id": "r:409", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:417" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:416" ] }, { "annotation": null, "id": "r:410", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:417" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:409", "results": [ "s:416" ] }, { "annotation": null, "id": "r:411", "op": "linear.concat", "operands": [ "s:100", "s:164" ], "params": {}, "parent": null, "results": [ "s:417" ] }, { "annotation": null, "id": "r:412", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:416" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": -0.5, "true_level": 0.0 }, "parent": null, "results": [ "s:418" ] }, { "annotation": null, "id": "r:413", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:416" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:412", "results": [ "s:418" ] }, { "annotation": null, "id": "r:414", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:420" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:419" ] }, { "annotation": null, "id": "r:415", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:420" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:414", "results": [ "s:419" ] }, { "annotation": null, "id": "r:416", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:421" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:420" ] }, { "annotation": null, "id": "r:417", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:421" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:416", "results": [ "s:420" ] }, { "annotation": null, "id": "r:418", "op": "linear.concat", "operands": [ "s:247", "s:228" ], "params": {}, "parent": null, "results": [ "s:421" ] }, { "annotation": null, "id": "r:419", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:420" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": -0.5, "true_level": 0.0 }, "parent": null, "results": [ "s:422" ] }, { "annotation": null, "id": "r:420", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:420" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:419", "results": [ "s:422" ] }, { "annotation": null, "id": "r:421", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:424" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:423" ] }, { "annotation": null, "id": "r:422", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:424" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:421", "results": [ "s:423" ] }, { "annotation": null, "id": "r:423", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:425" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:424" ] }, { "annotation": null, "id": "r:424", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:425" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:423", "results": [ "s:424" ] }, { "annotation": null, "id": "r:425", "op": "linear.concat", "operands": [ "s:294", "s:271" ], "params": {}, "parent": null, "results": [ "s:425" ] }, { "annotation": null, "id": "r:426", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:424" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": -0.5, "true_level": 0.0 }, "parent": null, "results": [ "s:426" ] }, { "annotation": null, "id": "r:427", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:424" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:426", "results": [ "s:426" ] }, { "annotation": null, "id": "r:428", "op": "swiglu.map_select.select", "operands": [ "s:412", "s:164", "s:100" ], "params": {}, "parent": null, "results": [ "s:427" ] }, { "annotation": null, "id": "r:429", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:428" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:428", "results": [ "s:427" ] }, { "annotation": null, "id": "r:430", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:430" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:429" ] }, { "annotation": null, "id": "r:431", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:430" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:430", "results": [ "s:429" ] }, { "annotation": null, "id": "r:432", "op": "linear.concat", "operands": [ "s:431", "s:433", "s:435" ], "params": {}, "parent": null, "results": [ "s:430" ] }, { "annotation": null, "id": "r:433", "op": "swiglu.map_select.select", "operands": [ "s:412", "s:247", "s:228" ], "params": {}, "parent": null, "results": [ "s:431" ] }, { "annotation": null, "id": "r:434", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:432" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:433", "results": [ "s:431" ] }, { "annotation": null, "id": "r:435", "op": "swiglu.map_select.select", "operands": [ "s:412", "s:228", "s:247" ], "params": {}, "parent": null, "results": [ "s:433" ] }, { "annotation": null, "id": "r:436", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:434" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:435", "results": [ "s:433" ] }, { "annotation": null, "id": "r:437", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:436" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:435" ] }, { "annotation": null, "id": "r:438", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:436" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:437", "results": [ "s:435" ] }, { "annotation": null, "id": "r:439", "op": "linear.concat", "operands": [ "s:437", "s:439", "s:441" ], "params": {}, "parent": null, "results": [ "s:436" ] }, { "annotation": null, "id": "r:440", "op": "swiglu.map_select.select", "operands": [ "s:412", "s:294", "s:271" ], "params": {}, "parent": null, "results": [ "s:437" ] }, { "annotation": null, "id": "r:441", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:438" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:440", "results": [ "s:437" ] }, { "annotation": null, "id": "r:442", "op": "swiglu.map_select.select", "operands": [ "s:412", "s:271", "s:294" ], "params": {}, "parent": null, "results": [ "s:439" ] }, { "annotation": null, "id": "r:443", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:440" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:442", "results": [ "s:439" ] }, { "annotation": null, "id": "r:444", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 2 ] } } }, "parent": null, "results": [ "s:441" ] }, { "annotation": null, "id": "r:445", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:444" ], "params": {}, "parent": "r:391", "results": [ "s:442" ] }, { "annotation": null, "id": "r:446", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:443" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:445", "results": [ "s:442" ] }, { "annotation": null, "id": "r:447", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:445" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:444" ] }, { "annotation": null, "id": "r:448", "op": "linear.bool_to_01", "operands": [ "s:447" ], "params": {}, "parent": null, "results": [ "s:445" ] }, { "annotation": null, "id": "r:449", "op": "linear.multiply_const", "operands": [ "s:446" ], "params": { "scalar": 0.5 }, "parent": "r:448", "results": [ "s:445" ] }, { "annotation": null, "id": "r:450", "op": "linear.add_const", "operands": [ "s:447" ], "params": { "scalar": 1.0 }, "parent": "r:448", "results": [ "s:446" ] }, { "annotation": null, "id": "r:451", "op": "swiglu.map_select.in_range", "operands": [ "s:311", "s:449" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:447" ] }, { "annotation": null, "id": "r:452", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:448" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:451", "results": [ "s:447" ] }, { "annotation": null, "id": "r:453", "op": "linear.add_const", "operands": [ "s:311" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:449" ] }, { "annotation": null, "id": "r:454", "op": "linear.bool_to_01", "operands": [ "s:452" ], "params": {}, "parent": "r:28", "results": [ "s:450" ] }, { "annotation": null, "id": "r:455", "op": "linear.multiply_const", "operands": [ "s:451" ], "params": { "scalar": 0.5 }, "parent": "r:454", "results": [ "s:450" ] }, { "annotation": null, "id": "r:456", "op": "linear.add_const", "operands": [ "s:452" ], "params": { "scalar": 1.0 }, "parent": "r:454", "results": [ "s:451" ] }, { "annotation": null, "id": "r:457", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:453" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 4.5, "true_level": 1.0 }, "parent": "r:28", "results": [ "s:452" ] }, { "annotation": null, "id": "r:458", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:453" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:457", "results": [ "s:452" ] }, { "annotation": null, "id": "r:459", "op": "linear.sum_nodes", "operands": [ "s:39", "s:344", "s:366", "s:398", "s:455" ], "params": { "max_fanout": null }, "parent": "r:28", "results": [ "s:453" ] }, { "annotation": null, "id": "r:460", "op": "linear.bool_to_01", "operands": [ "s:457" ], "params": {}, "parent": "r:28", "results": [ "s:455" ] }, { "annotation": null, "id": "r:461", "op": "linear.multiply_const", "operands": [ "s:456" ], "params": { "scalar": 0.5 }, "parent": "r:460", "results": [ "s:455" ] }, { "annotation": null, "id": "r:462", "op": "linear.add_const", "operands": [ "s:457" ], "params": { "scalar": 1.0 }, "parent": "r:460", "results": [ "s:456" ] }, { "annotation": null, "id": "r:463", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:458" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:28", "results": [ "s:457" ] }, { "annotation": null, "id": "r:464", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:458" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:463", "results": [ "s:457" ] }, { "annotation": null, "id": "r:465", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:462", "s:465", "s:468" ], "params": {}, "parent": null, "results": [ "s:458" ] }, { "annotation": null, "id": "r:466", "op": "linear.sum_nodes", "operands": [ "s:460", "s:463", "s:466" ], "params": { "max_fanout": null }, "parent": "r:465", "results": [ "s:458" ] }, { "annotation": null, "id": "r:467", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:462" ], "params": {}, "parent": "r:465", "results": [ "s:460" ] }, { "annotation": null, "id": "r:468", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:461" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:467", "results": [ "s:460" ] }, { "annotation": null, "id": "r:469", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:379" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:462" ] }, { "annotation": null, "id": "r:470", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:379" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:469", "results": [ "s:462" ] }, { "annotation": null, "id": "r:471", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:465" ], "params": {}, "parent": "r:465", "results": [ "s:463" ] }, { "annotation": null, "id": "r:472", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:464" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:471", "results": [ "s:463" ] }, { "annotation": null, "id": "r:473", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:430" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:465" ] }, { "annotation": null, "id": "r:474", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:430" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:473", "results": [ "s:465" ] }, { "annotation": null, "id": "r:475", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:468" ], "params": {}, "parent": "r:465", "results": [ "s:466" ] }, { "annotation": null, "id": "r:476", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:467" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:475", "results": [ "s:466" ] }, { "annotation": null, "id": "r:477", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:469" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:468" ] }, { "annotation": null, "id": "r:478", "op": "linear.bool_to_01", "operands": [ "s:471" ], "params": {}, "parent": null, "results": [ "s:469" ] }, { "annotation": null, "id": "r:479", "op": "linear.multiply_const", "operands": [ "s:470" ], "params": { "scalar": 0.5 }, "parent": "r:478", "results": [ "s:469" ] }, { "annotation": null, "id": "r:480", "op": "linear.add_const", "operands": [ "s:471" ], "params": { "scalar": 1.0 }, "parent": "r:478", "results": [ "s:470" ] }, { "annotation": null, "id": "r:481", "op": "swiglu.map_select.in_range", "operands": [ "s:324", "s:473" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:471" ] }, { "annotation": null, "id": "r:482", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:472" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:481", "results": [ "s:471" ] }, { "annotation": null, "id": "r:483", "op": "linear.add_const", "operands": [ "s:324" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:473" ] }, { "annotation": null, "id": "r:484", "op": "linear.multiply_const", "operands": [ "s:475" ], "params": { "scalar": 6.0 }, "parent": "r:28", "results": [ "s:474" ] }, { "annotation": null, "id": "r:485", "op": "linear.bool_to_01", "operands": [ "s:477" ], "params": {}, "parent": "r:28", "results": [ "s:475" ] }, { "annotation": null, "id": "r:486", "op": "linear.multiply_const", "operands": [ "s:476" ], "params": { "scalar": 0.5 }, "parent": "r:485", "results": [ "s:475" ] }, { "annotation": null, "id": "r:487", "op": "linear.add_const", "operands": [ "s:477" ], "params": { "scalar": 1.0 }, "parent": "r:485", "results": [ "s:476" ] }, { "annotation": null, "id": "r:488", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:74", "s:480" ], "params": {}, "parent": null, "results": [ "s:477" ] }, { "annotation": null, "id": "r:489", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:478" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:488", "results": [ "s:477" ] }, { "annotation": null, "id": "r:490", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:478" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:489", "results": [ "s:477" ] }, { "annotation": null, "id": "r:491", "op": "linear.sum_nodes", "operands": [ "s:74", "s:480" ], "params": { "max_fanout": null }, "parent": "r:488", "results": [ "s:478" ] }, { "annotation": null, "id": "r:492", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:412" ], "params": {}, "parent": null, "results": [ "s:480" ] }, { "annotation": null, "id": "r:493", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:412" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:492", "results": [ "s:480" ] }, { "annotation": null, "id": "r:494", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:412" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:493", "results": [ "s:480" ] }, { "annotation": null, "id": "r:495", "op": "linear.add_const", "operands": [ "s:31" ], "params": { "scalar": 1.0 }, "parent": "r:28", "results": [ "s:481" ] }, { "annotation": null, "id": "r:496", "op": "linear.concat", "operands": [ "s:42", "s:347", "s:369", "s:401", "s:458", "s:483", "s:499", "s:499", "s:499", "s:499", "s:499", "s:499" ], "params": {}, "parent": "r:28", "results": [ "s:482" ] }, { "annotation": null, "id": "r:497", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:487", "s:490", "s:493" ], "params": {}, "parent": null, "results": [ "s:483" ] }, { "annotation": null, "id": "r:498", "op": "linear.sum_nodes", "operands": [ "s:485", "s:488", "s:491" ], "params": { "max_fanout": null }, "parent": "r:497", "results": [ "s:483" ] }, { "annotation": null, "id": "r:499", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:487" ], "params": {}, "parent": "r:497", "results": [ "s:485" ] }, { "annotation": null, "id": "r:500", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:486" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:499", "results": [ "s:485" ] }, { "annotation": null, "id": "r:501", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:381" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:487" ] }, { "annotation": null, "id": "r:502", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:381" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:501", "results": [ "s:487" ] }, { "annotation": null, "id": "r:503", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:490" ], "params": {}, "parent": "r:497", "results": [ "s:488" ] }, { "annotation": null, "id": "r:504", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:489" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:503", "results": [ "s:488" ] }, { "annotation": null, "id": "r:505", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:436" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:490" ] }, { "annotation": null, "id": "r:506", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:436" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:505", "results": [ "s:490" ] }, { "annotation": null, "id": "r:507", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:493" ], "params": {}, "parent": "r:497", "results": [ "s:491" ] }, { "annotation": null, "id": "r:508", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:492" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:507", "results": [ "s:491" ] }, { "annotation": null, "id": "r:509", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:494" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:493" ] }, { "annotation": null, "id": "r:510", "op": "linear.bool_to_01", "operands": [ "s:496" ], "params": {}, "parent": null, "results": [ "s:494" ] }, { "annotation": null, "id": "r:511", "op": "linear.multiply_const", "operands": [ "s:495" ], "params": { "scalar": 0.5 }, "parent": "r:510", "results": [ "s:494" ] }, { "annotation": null, "id": "r:512", "op": "linear.add_const", "operands": [ "s:496" ], "params": { "scalar": 1.0 }, "parent": "r:510", "results": [ "s:495" ] }, { "annotation": null, "id": "r:513", "op": "swiglu.map_select.in_range", "operands": [ "s:333", "s:498" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:496" ] }, { "annotation": null, "id": "r:514", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:497" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:513", "results": [ "s:496" ] }, { "annotation": null, "id": "r:515", "op": "linear.add_const", "operands": [ "s:333" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:498" ] }, { "annotation": null, "id": "r:516", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "remove_leading_0s_sign", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:28", "results": [ "s:499" ] }, { "annotation": null, "id": "r:517", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:502", "s:508" ], "params": {}, "parent": "r:0", "results": [ "s:500" ] }, { "annotation": null, "id": "r:518", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:501" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:517", "results": [ "s:500" ] }, { "annotation": null, "id": "r:519", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:505", "s:506" ], "params": {}, "parent": "r:0", "results": [ "s:502" ] }, { "annotation": null, "id": "r:520", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:503" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:519", "results": [ "s:502" ] }, { "annotation": null, "id": "r:521", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:503" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:520", "results": [ "s:502" ] }, { "annotation": null, "id": "r:522", "op": "linear.sum_nodes", "operands": [ "s:505", "s:506" ], "params": { "max_fanout": null }, "parent": "r:519", "results": [ "s:503" ] }, { "annotation": null, "id": "r:523", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:505" ] }, { "annotation": null, "id": "r:524", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:523", "results": [ "s:505" ] }, { "annotation": null, "id": "r:525", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:507" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -1.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:506" ] }, { "annotation": null, "id": "r:526", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:507" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:525", "results": [ "s:506" ] }, { "annotation": null, "id": "r:527", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:507" ] }, { "annotation": null, "id": "r:528", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:511" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:509" ] }, { "annotation": null, "id": "r:529", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:510" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:528", "results": [ "s:509" ] }, { "annotation": null, "id": "r:530", "op": "linear.concat", "operands": [ "s:347", "s:369", "s:401", "s:458", "s:483", "s:512", "s:42", "s:347", "s:369", "s:401", "s:458", "s:483" ], "params": {}, "parent": "r:28", "results": [ "s:511" ] }, { "annotation": null, "id": "r:531", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:516", "s:519", "s:522" ], "params": {}, "parent": null, "results": [ "s:512" ] }, { "annotation": null, "id": "r:532", "op": "linear.sum_nodes", "operands": [ "s:514", "s:517", "s:520" ], "params": { "max_fanout": null }, "parent": "r:531", "results": [ "s:512" ] }, { "annotation": null, "id": "r:533", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:516" ], "params": {}, "parent": "r:531", "results": [ "s:514" ] }, { "annotation": null, "id": "r:534", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:515" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:533", "results": [ "s:514" ] }, { "annotation": null, "id": "r:535", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:516" ] }, { "annotation": null, "id": "r:536", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:519" ], "params": {}, "parent": "r:531", "results": [ "s:517" ] }, { "annotation": null, "id": "r:537", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:518" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:536", "results": [ "s:517" ] }, { "annotation": null, "id": "r:538", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:519" ] }, { "annotation": null, "id": "r:539", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:522" ], "params": {}, "parent": "r:531", "results": [ "s:520" ] }, { "annotation": null, "id": "r:540", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:521" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:539", "results": [ "s:520" ] }, { "annotation": null, "id": "r:541", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:522" ] }, { "annotation": null, "id": "r:542", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:525", "s:531" ], "params": {}, "parent": "r:0", "results": [ "s:523" ] }, { "annotation": null, "id": "r:543", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:524" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:542", "results": [ "s:523" ] }, { "annotation": null, "id": "r:544", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:528", "s:529" ], "params": {}, "parent": "r:0", "results": [ "s:525" ] }, { "annotation": null, "id": "r:545", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:526" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:544", "results": [ "s:525" ] }, { "annotation": null, "id": "r:546", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:526" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:545", "results": [ "s:525" ] }, { "annotation": null, "id": "r:547", "op": "linear.sum_nodes", "operands": [ "s:528", "s:529" ], "params": { "max_fanout": null }, "parent": "r:544", "results": [ "s:526" ] }, { "annotation": null, "id": "r:548", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:528" ] }, { "annotation": null, "id": "r:549", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:548", "results": [ "s:528" ] }, { "annotation": null, "id": "r:550", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:530" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -2.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:529" ] }, { "annotation": null, "id": "r:551", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:530" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:550", "results": [ "s:529" ] }, { "annotation": null, "id": "r:552", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:530" ] }, { "annotation": null, "id": "r:553", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:534" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:532" ] }, { "annotation": null, "id": "r:554", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:533" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:553", "results": [ "s:532" ] }, { "annotation": null, "id": "r:555", "op": "linear.concat", "operands": [ "s:369", "s:401", "s:458", "s:483", "s:512", "s:535", "s:347", "s:369", "s:401", "s:458", "s:483", "s:512" ], "params": {}, "parent": "r:28", "results": [ "s:534" ] }, { "annotation": null, "id": "r:556", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:516", "s:519", "s:522" ], "params": {}, "parent": null, "results": [ "s:535" ] }, { "annotation": null, "id": "r:557", "op": "linear.sum_nodes", "operands": [ "s:537", "s:539", "s:541" ], "params": { "max_fanout": null }, "parent": "r:556", "results": [ "s:535" ] }, { "annotation": null, "id": "r:558", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:516" ], "params": {}, "parent": "r:556", "results": [ "s:537" ] }, { "annotation": null, "id": "r:559", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:538" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:558", "results": [ "s:537" ] }, { "annotation": null, "id": "r:560", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:519" ], "params": {}, "parent": "r:556", "results": [ "s:539" ] }, { "annotation": null, "id": "r:561", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:540" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:560", "results": [ "s:539" ] }, { "annotation": null, "id": "r:562", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:522" ], "params": {}, "parent": "r:556", "results": [ "s:541" ] }, { "annotation": null, "id": "r:563", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:542" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:562", "results": [ "s:541" ] }, { "annotation": null, "id": "r:564", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:545", "s:551" ], "params": {}, "parent": "r:0", "results": [ "s:543" ] }, { "annotation": null, "id": "r:565", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:544" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:564", "results": [ "s:543" ] }, { "annotation": null, "id": "r:566", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:548", "s:549" ], "params": {}, "parent": "r:0", "results": [ "s:545" ] }, { "annotation": null, "id": "r:567", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:546" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:566", "results": [ "s:545" ] }, { "annotation": null, "id": "r:568", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:546" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:567", "results": [ "s:545" ] }, { "annotation": null, "id": "r:569", "op": "linear.sum_nodes", "operands": [ "s:548", "s:549" ], "params": { "max_fanout": null }, "parent": "r:566", "results": [ "s:546" ] }, { "annotation": null, "id": "r:570", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 2.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:548" ] }, { "annotation": null, "id": "r:571", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:570", "results": [ "s:548" ] }, { "annotation": null, "id": "r:572", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:550" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -3.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:549" ] }, { "annotation": null, "id": "r:573", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:550" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:572", "results": [ "s:549" ] }, { "annotation": null, "id": "r:574", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:550" ] }, { "annotation": null, "id": "r:575", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:554" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:552" ] }, { "annotation": null, "id": "r:576", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:553" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:575", "results": [ "s:552" ] }, { "annotation": null, "id": "r:577", "op": "linear.concat", "operands": [ "s:401", "s:458", "s:483", "s:512", "s:535", "s:535", "s:369", "s:401", "s:458", "s:483", "s:512", "s:535" ], "params": {}, "parent": "r:28", "results": [ "s:554" ] }, { "annotation": null, "id": "r:578", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:557", "s:563" ], "params": {}, "parent": "r:0", "results": [ "s:555" ] }, { "annotation": null, "id": "r:579", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:556" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:578", "results": [ "s:555" ] }, { "annotation": null, "id": "r:580", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:560", "s:561" ], "params": {}, "parent": "r:0", "results": [ "s:557" ] }, { "annotation": null, "id": "r:581", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:558" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:580", "results": [ "s:557" ] }, { "annotation": null, "id": "r:582", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:558" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:581", "results": [ "s:557" ] }, { "annotation": null, "id": "r:583", "op": "linear.sum_nodes", "operands": [ "s:560", "s:561" ], "params": { "max_fanout": null }, "parent": "r:580", "results": [ "s:558" ] }, { "annotation": null, "id": "r:584", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 3.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:560" ] }, { "annotation": null, "id": "r:585", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:584", "results": [ "s:560" ] }, { "annotation": null, "id": "r:586", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:562" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -4.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:561" ] }, { "annotation": null, "id": "r:587", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:562" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:586", "results": [ "s:561" ] }, { "annotation": null, "id": "r:588", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:562" ] }, { "annotation": null, "id": "r:589", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:566" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:564" ] }, { "annotation": null, "id": "r:590", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:565" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:589", "results": [ "s:564" ] }, { "annotation": null, "id": "r:591", "op": "linear.concat", "operands": [ "s:458", "s:483", "s:512", "s:535", "s:535", "s:535", "s:401", "s:458", "s:483", "s:512", "s:535", "s:535" ], "params": {}, "parent": "r:28", "results": [ "s:566" ] }, { "annotation": null, "id": "r:592", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:569", "s:575" ], "params": {}, "parent": "r:0", "results": [ "s:567" ] }, { "annotation": null, "id": "r:593", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:568" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:592", "results": [ "s:567" ] }, { "annotation": null, "id": "r:594", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:572", "s:573" ], "params": {}, "parent": "r:0", "results": [ "s:569" ] }, { "annotation": null, "id": "r:595", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:570" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:594", "results": [ "s:569" ] }, { "annotation": null, "id": "r:596", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:570" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:595", "results": [ "s:569" ] }, { "annotation": null, "id": "r:597", "op": "linear.sum_nodes", "operands": [ "s:572", "s:573" ], "params": { "max_fanout": null }, "parent": "r:594", "results": [ "s:570" ] }, { "annotation": null, "id": "r:598", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 4.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:572" ] }, { "annotation": null, "id": "r:599", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:598", "results": [ "s:572" ] }, { "annotation": null, "id": "r:600", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:574" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -5.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:573" ] }, { "annotation": null, "id": "r:601", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:574" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:600", "results": [ "s:573" ] }, { "annotation": null, "id": "r:602", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:574" ] }, { "annotation": null, "id": "r:603", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:578" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:576" ] }, { "annotation": null, "id": "r:604", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:577" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:603", "results": [ "s:576" ] }, { "annotation": null, "id": "r:605", "op": "linear.concat", "operands": [ "s:483", "s:512", "s:535", "s:535", "s:535", "s:535", "s:458", "s:483", "s:512", "s:535", "s:535", "s:535" ], "params": {}, "parent": "r:28", "results": [ "s:578" ] }, { "annotation": null, "id": "r:606", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:581", "s:587" ], "params": {}, "parent": "r:0", "results": [ "s:579" ] }, { "annotation": null, "id": "r:607", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:580" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:606", "results": [ "s:579" ] }, { "annotation": null, "id": "r:608", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:584", "s:585" ], "params": {}, "parent": "r:0", "results": [ "s:581" ] }, { "annotation": null, "id": "r:609", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:582" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:608", "results": [ "s:581" ] }, { "annotation": null, "id": "r:610", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:582" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:609", "results": [ "s:581" ] }, { "annotation": null, "id": "r:611", "op": "linear.sum_nodes", "operands": [ "s:584", "s:585" ], "params": { "max_fanout": null }, "parent": "r:608", "results": [ "s:582" ] }, { "annotation": null, "id": "r:612", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 5.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:584" ] }, { "annotation": null, "id": "r:613", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:612", "results": [ "s:584" ] }, { "annotation": null, "id": "r:614", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:586" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -6.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:585" ] }, { "annotation": null, "id": "r:615", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:586" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:614", "results": [ "s:585" ] }, { "annotation": null, "id": "r:616", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:586" ] }, { "annotation": null, "id": "r:617", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:590" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:588" ] }, { "annotation": null, "id": "r:618", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:589" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:617", "results": [ "s:588" ] }, { "annotation": null, "id": "r:619", "op": "linear.concat", "operands": [ "s:512", "s:535", "s:535", "s:535", "s:535", "s:535", "s:483", "s:512", "s:535", "s:535", "s:535", "s:535" ], "params": {}, "parent": "r:28", "results": [ "s:590" ] }, { "annotation": null, "id": "r:620", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:593", "s:599" ], "params": {}, "parent": "r:0", "results": [ "s:591" ] }, { "annotation": null, "id": "r:621", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:592" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:620", "results": [ "s:591" ] }, { "annotation": null, "id": "r:622", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:596", "s:597" ], "params": {}, "parent": "r:0", "results": [ "s:593" ] }, { "annotation": null, "id": "r:623", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:594" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:622", "results": [ "s:593" ] }, { "annotation": null, "id": "r:624", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:594" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:623", "results": [ "s:593" ] }, { "annotation": null, "id": "r:625", "op": "linear.sum_nodes", "operands": [ "s:596", "s:597" ], "params": { "max_fanout": null }, "parent": "r:622", "results": [ "s:594" ] }, { "annotation": null, "id": "r:626", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 6.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:596" ] }, { "annotation": null, "id": "r:627", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:626", "results": [ "s:596" ] }, { "annotation": null, "id": "r:628", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:598" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -7.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:597" ] }, { "annotation": null, "id": "r:629", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:598" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:628", "results": [ "s:597" ] }, { "annotation": null, "id": "r:630", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:598" ] }, { "annotation": null, "id": "r:631", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:602" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:600" ] }, { "annotation": null, "id": "r:632", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:601" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:631", "results": [ "s:600" ] }, { "annotation": null, "id": "r:633", "op": "linear.concat", "operands": [ "s:535", "s:535", "s:535", "s:535", "s:535", "s:535", "s:512", "s:535", "s:535", "s:535", "s:535", "s:535" ], "params": {}, "parent": "r:28", "results": [ "s:602" ] }, { "annotation": null, "id": "r:634", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:0", "results": [ "s:603" ] } ], "sha256": "5fdcd7851e4967727f32dba4167de8b460fd7f1e16b4e788a35d3967b2b546c0" } }, "integrity": { "scope": "every manifest section except this one", "sha256": "366e8133d899aa41bef251ada809f0b195617853fb119e43b1c8906241e4c3a2" }, "intervention_contract": { "guarantee": "The manifest describes the unmodified compiled program. Interventions outside a node's stated value contract are not covered by torchwright's semantic or numerical guarantees.", "node_value_contracts": "graphs.source.nodes[*].value_contract", "physical_coordinates": { "attention_heads": "schedule.layers[*].attention_operations[*].heads", "mlp_neurons": "schedule.layers[*].mlp_operations[*].hidden_slots", "residual": "schedule.layers[*].attention_operations[*] and mlp_operations[*] target/source/query/key column runs" } }, "metadata": { "source": { "arguments": { "max_digits": 3 }, "entrypoint": "examples.calculator_simple:create_network_parts" }, "task": { "example": "calculator_simple", "max_digits": 3 } }, "model": { "activation": "swish", "active_attention_heads_per_layer": [ 1, 4, 3, 3, 8, 6, 2, 4, 5, 1, 4, 8, 2, 3, 2, 2, 2, 3, 4, 4, 2, 1, 2, 1, 1, 32, 5 ], "active_mlp_neurons_per_layer": [ 16, 35, 73, 25, 310, 366, 84, 56, 22, 7, 9, 1605, 501, 643, 922, 797, 789, 676, 180, 80, 23, 13, 5, 47, 1963, 3415, 479 ], "architecture": "phi3", "bias": false, "d_head": 64, "d_model": 2048, "max_sequence_length": 512, "n_layers": 27, "rms_norm": true, "rms_norm_eps": 1e-05, "rope_base": 500000.0, "rotary_width": 64, "stored_attention_heads": 32, "stored_mlp_neurons": 3415 }, "observability": { "abstract_residual_states": [ { "hook": "blocks.0.hook_resid_pre", "state": "input" }, { "hook": "blocks.0.hook_resid_post", "state": "L0.post" }, { "hook": "blocks.1.hook_resid_post", "state": "L1.post" }, { "hook": "blocks.2.hook_resid_post", "state": "L2.post" }, { "hook": "blocks.3.hook_resid_post", "state": "L3.post" }, { "hook": "blocks.4.hook_resid_post", "state": "L4.post" }, { "hook": "blocks.5.hook_resid_post", "state": "L5.post" }, { "hook": "blocks.6.hook_resid_post", "state": "L6.post" }, { "hook": "blocks.7.hook_resid_post", "state": "L7.post" }, { "hook": "blocks.8.hook_resid_post", "state": "L8.post" }, { "hook": "blocks.9.hook_resid_post", "state": "L9.post" }, { "hook": "blocks.10.hook_resid_post", "state": "L10.post" }, { "hook": "blocks.11.hook_resid_post", "state": "L11.post" }, { "hook": "blocks.12.hook_resid_post", "state": "L12.post" }, { "hook": "blocks.13.hook_resid_post", "state": "L13.post" }, { "hook": "blocks.14.hook_resid_post", "state": "L14.post" }, { "hook": "blocks.15.hook_resid_post", "state": "L15.post" }, { "hook": "blocks.16.hook_resid_post", "state": "L16.post" }, { "hook": "blocks.17.hook_resid_post", "state": "L17.post" }, { "hook": "blocks.18.hook_resid_post", "state": "L18.post" }, { "hook": "blocks.19.hook_resid_post", "state": "L19.post" }, { "hook": "blocks.20.hook_resid_post", "state": "L20.post" }, { "hook": "blocks.21.hook_resid_post", "state": "L21.post" }, { "hook": "blocks.22.hook_resid_post", "state": "L22.post" }, { "hook": "blocks.23.hook_resid_post", "state": "L23.post" }, { "hook": "blocks.24.hook_resid_post", "state": "L24.post" }, { "hook": "blocks.25.hook_resid_post", "state": "L25.post" }, { "hook": "blocks.26.hook_resid_post", "state": "L26.post" } ], "transformer_lens": { "attention": { "head_axis": 2, "k": "blocks.{layer}.attn.hook_k", "pattern": "blocks.{layer}.attn.hook_pattern", "q": "blocks.{layer}.attn.hook_q", "result": "blocks.{layer}.attn.hook_result", "v": "blocks.{layer}.attn.hook_v" }, "mlp": { "neuron_axis": -1, "post": "blocks.{layer}.mlp.hook_post", "pre": "blocks.{layer}.mlp.hook_pre" }, "note": "Use schedule.layers head and hidden-slot ranges to slice the standard hooks. Hook availability depends on the adapter version.", "residual_states": [ { "hook": "blocks.0.hook_resid_pre", "state": "input" }, { "hook": "blocks.0.hook_resid_post", "state": "L0.post" }, { "hook": "blocks.1.hook_resid_post", "state": "L1.post" }, { "hook": "blocks.2.hook_resid_post", "state": "L2.post" }, { "hook": "blocks.3.hook_resid_post", "state": "L3.post" }, { "hook": "blocks.4.hook_resid_post", "state": "L4.post" }, { "hook": "blocks.5.hook_resid_post", "state": "L5.post" }, { "hook": "blocks.6.hook_resid_post", "state": "L6.post" }, { "hook": "blocks.7.hook_resid_post", "state": "L7.post" }, { "hook": "blocks.8.hook_resid_post", "state": "L8.post" }, { "hook": "blocks.9.hook_resid_post", "state": "L9.post" }, { "hook": "blocks.10.hook_resid_post", "state": "L10.post" }, { "hook": "blocks.11.hook_resid_post", "state": "L11.post" }, { "hook": "blocks.12.hook_resid_post", "state": "L12.post" }, { "hook": "blocks.13.hook_resid_post", "state": "L13.post" }, { "hook": "blocks.14.hook_resid_post", "state": "L14.post" }, { "hook": "blocks.15.hook_resid_post", "state": "L15.post" }, { "hook": "blocks.16.hook_resid_post", "state": "L16.post" }, { "hook": "blocks.17.hook_resid_post", "state": "L17.post" }, { "hook": "blocks.18.hook_resid_post", "state": "L18.post" }, { "hook": "blocks.19.hook_resid_post", "state": "L19.post" }, { "hook": "blocks.20.hook_resid_post", "state": "L20.post" }, { "hook": "blocks.21.hook_resid_post", "state": "L21.post" }, { "hook": "blocks.22.hook_resid_post", "state": "L22.post" }, { "hook": "blocks.23.hook_resid_post", "state": "L23.post" }, { "hook": "blocks.24.hook_resid_post", "state": "L24.post" }, { "hook": "blocks.25.hook_resid_post", "state": "L25.post" }, { "hook": "blocks.26.hook_resid_post", "state": "L26.post" } ] } }, "parameter_support": { "file": "torchwright_schematic_support.npz", "format": "torchwright.csr_support.v1", "sha256": "ba74247d8781a6aa68ee715776792e9a4ab703b74732941d3a81cb4147735615", "tensors": { "model.embed_tokens.weight": { "chunks": [ { "indices": "tensor_000108_chunk_000000_indices", "indptr": "tensor_000108_chunk_000000_indptr", "nnz": 71, "row_count": 18, "row_start": 0 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 71, "shape": [ 18, 2048 ] }, "model.layers.0.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000110_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.0.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000003_chunk_000000_indices", "indptr": "tensor_000003_chunk_000000_indptr", "nnz": 72, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000003_chunk_000001_indices", "indptr": "tensor_000003_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000003_chunk_000002_indices", "indptr": "tensor_000003_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000003_chunk_000003_indices", "indptr": "tensor_000003_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000003_chunk_000004_indices", "indptr": "tensor_000003_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000003_chunk_000005_indices", "indptr": "tensor_000003_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000003_chunk_000006_indices", "indptr": "tensor_000003_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 72, "shape": [ 2048, 3415 ] }, "model.layers.0.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000002_chunk_000000_indices", "indptr": "tensor_000002_chunk_000000_indptr", "nnz": 16, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000002_chunk_000001_indices", "indptr": "tensor_000002_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000002_chunk_000002_indices", "indptr": "tensor_000002_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000002_chunk_000003_indices", "indptr": "tensor_000002_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000002_chunk_000004_indices", "indptr": "tensor_000002_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000002_chunk_000005_indices", "indptr": "tensor_000002_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000002_chunk_000006_indices", "indptr": "tensor_000002_chunk_000006_indptr", "nnz": 16, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000002_chunk_000007_indices", "indptr": "tensor_000002_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000002_chunk_000008_indices", "indptr": "tensor_000002_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000002_chunk_000009_indices", "indptr": "tensor_000002_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000002_chunk_000010_indices", "indptr": "tensor_000002_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000002_chunk_000011_indices", "indptr": "tensor_000002_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000002_chunk_000012_indices", "indptr": "tensor_000002_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000002_chunk_000013_indices", "indptr": "tensor_000002_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 32, "shape": [ 6830, 2048 ] }, "model.layers.0.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000111_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.0.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000001_chunk_000000_indices", "indptr": "tensor_000001_chunk_000000_indptr", "nnz": 0, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000001_chunk_000001_indices", "indptr": "tensor_000001_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000001_chunk_000002_indices", "indptr": "tensor_000001_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000001_chunk_000003_indices", "indptr": "tensor_000001_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 0, "shape": [ 2048, 2048 ] }, "model.layers.0.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000000_chunk_000000_indices", "indptr": "tensor_000000_chunk_000000_indptr", "nnz": 0, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000000_chunk_000001_indices", "indptr": "tensor_000000_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000000_chunk_000002_indices", "indptr": "tensor_000000_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000000_chunk_000003_indices", "indptr": "tensor_000000_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000000_chunk_000004_indices", "indptr": "tensor_000000_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000000_chunk_000005_indices", "indptr": "tensor_000000_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000000_chunk_000006_indices", "indptr": "tensor_000000_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000000_chunk_000007_indices", "indptr": "tensor_000000_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000000_chunk_000008_indices", "indptr": "tensor_000000_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000000_chunk_000009_indices", "indptr": "tensor_000000_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000000_chunk_000010_indices", "indptr": "tensor_000000_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000000_chunk_000011_indices", "indptr": "tensor_000000_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 0, "shape": [ 6144, 2048 ] }, "model.layers.1.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000112_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.1.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000007_chunk_000000_indices", "indptr": "tensor_000007_chunk_000000_indptr", "nnz": 39, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000007_chunk_000001_indices", "indptr": "tensor_000007_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000007_chunk_000002_indices", "indptr": "tensor_000007_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000007_chunk_000003_indices", "indptr": "tensor_000007_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000007_chunk_000004_indices", "indptr": "tensor_000007_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000007_chunk_000005_indices", "indptr": "tensor_000007_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000007_chunk_000006_indices", "indptr": "tensor_000007_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 39, "shape": [ 2048, 3415 ] }, "model.layers.1.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000006_chunk_000000_indices", "indptr": "tensor_000006_chunk_000000_indptr", "nnz": 40, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000006_chunk_000001_indices", "indptr": "tensor_000006_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000006_chunk_000002_indices", "indptr": "tensor_000006_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000006_chunk_000003_indices", "indptr": "tensor_000006_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000006_chunk_000004_indices", "indptr": "tensor_000006_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000006_chunk_000005_indices", "indptr": "tensor_000006_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000006_chunk_000006_indices", "indptr": "tensor_000006_chunk_000006_indptr", "nnz": 35, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000006_chunk_000007_indices", "indptr": "tensor_000006_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000006_chunk_000008_indices", "indptr": "tensor_000006_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000006_chunk_000009_indices", "indptr": "tensor_000006_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000006_chunk_000010_indices", "indptr": "tensor_000006_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000006_chunk_000011_indices", "indptr": "tensor_000006_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000006_chunk_000012_indices", "indptr": "tensor_000006_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000006_chunk_000013_indices", "indptr": "tensor_000006_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 75, "shape": [ 6830, 2048 ] }, "model.layers.1.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000113_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.1.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000005_chunk_000000_indices", "indptr": "tensor_000005_chunk_000000_indptr", "nnz": 9, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000005_chunk_000001_indices", "indptr": "tensor_000005_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000005_chunk_000002_indices", "indptr": "tensor_000005_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000005_chunk_000003_indices", "indptr": "tensor_000005_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 9, "shape": [ 2048, 2048 ] }, "model.layers.1.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000004_chunk_000000_indices", "indptr": "tensor_000004_chunk_000000_indptr", "nnz": 97, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000004_chunk_000001_indices", "indptr": "tensor_000004_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000004_chunk_000002_indices", "indptr": "tensor_000004_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000004_chunk_000003_indices", "indptr": "tensor_000004_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000004_chunk_000004_indices", "indptr": "tensor_000004_chunk_000004_indptr", "nnz": 97, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000004_chunk_000005_indices", "indptr": "tensor_000004_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000004_chunk_000006_indices", "indptr": "tensor_000004_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000004_chunk_000007_indices", "indptr": "tensor_000004_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000004_chunk_000008_indices", "indptr": "tensor_000004_chunk_000008_indptr", "nnz": 9, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000004_chunk_000009_indices", "indptr": "tensor_000004_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000004_chunk_000010_indices", "indptr": "tensor_000004_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000004_chunk_000011_indices", "indptr": "tensor_000004_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 203, "shape": [ 6144, 2048 ] }, "model.layers.10.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000130_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.10.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000043_chunk_000000_indices", "indptr": "tensor_000043_chunk_000000_indptr", "nnz": 9, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000043_chunk_000001_indices", "indptr": "tensor_000043_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000043_chunk_000002_indices", "indptr": "tensor_000043_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000043_chunk_000003_indices", "indptr": "tensor_000043_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000043_chunk_000004_indices", "indptr": "tensor_000043_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000043_chunk_000005_indices", "indptr": "tensor_000043_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000043_chunk_000006_indices", "indptr": "tensor_000043_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 9, "shape": [ 2048, 3415 ] }, "model.layers.10.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000042_chunk_000000_indices", "indptr": "tensor_000042_chunk_000000_indptr", "nnz": 9, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000042_chunk_000001_indices", "indptr": "tensor_000042_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000042_chunk_000002_indices", "indptr": "tensor_000042_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000042_chunk_000003_indices", "indptr": "tensor_000042_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000042_chunk_000004_indices", "indptr": "tensor_000042_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000042_chunk_000005_indices", "indptr": "tensor_000042_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000042_chunk_000006_indices", "indptr": "tensor_000042_chunk_000006_indptr", "nnz": 9, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000042_chunk_000007_indices", "indptr": "tensor_000042_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000042_chunk_000008_indices", "indptr": "tensor_000042_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000042_chunk_000009_indices", "indptr": "tensor_000042_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000042_chunk_000010_indices", "indptr": "tensor_000042_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000042_chunk_000011_indices", "indptr": "tensor_000042_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000042_chunk_000012_indices", "indptr": "tensor_000042_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000042_chunk_000013_indices", "indptr": "tensor_000042_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 18, "shape": [ 6830, 2048 ] }, "model.layers.10.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000131_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.10.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000041_chunk_000000_indices", "indptr": "tensor_000041_chunk_000000_indptr", "nnz": 84, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000041_chunk_000001_indices", "indptr": "tensor_000041_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000041_chunk_000002_indices", "indptr": "tensor_000041_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000041_chunk_000003_indices", "indptr": "tensor_000041_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 84, "shape": [ 2048, 2048 ] }, "model.layers.10.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000040_chunk_000000_indices", "indptr": "tensor_000040_chunk_000000_indptr", "nnz": 76, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000040_chunk_000001_indices", "indptr": "tensor_000040_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000040_chunk_000002_indices", "indptr": "tensor_000040_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000040_chunk_000003_indices", "indptr": "tensor_000040_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000040_chunk_000004_indices", "indptr": "tensor_000040_chunk_000004_indptr", "nnz": 76, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000040_chunk_000005_indices", "indptr": "tensor_000040_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000040_chunk_000006_indices", "indptr": "tensor_000040_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000040_chunk_000007_indices", "indptr": "tensor_000040_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000040_chunk_000008_indices", "indptr": "tensor_000040_chunk_000008_indptr", "nnz": 84, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000040_chunk_000009_indices", "indptr": "tensor_000040_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000040_chunk_000010_indices", "indptr": "tensor_000040_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000040_chunk_000011_indices", "indptr": "tensor_000040_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 236, "shape": [ 6144, 2048 ] }, "model.layers.11.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000132_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.11.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000047_chunk_000000_indices", "indptr": "tensor_000047_chunk_000000_indptr", "nnz": 2015, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000047_chunk_000001_indices", "indptr": "tensor_000047_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000047_chunk_000002_indices", "indptr": "tensor_000047_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000047_chunk_000003_indices", "indptr": "tensor_000047_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000047_chunk_000004_indices", "indptr": "tensor_000047_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000047_chunk_000005_indices", "indptr": "tensor_000047_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000047_chunk_000006_indices", "indptr": "tensor_000047_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2015, "shape": [ 2048, 3415 ] }, "model.layers.11.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000046_chunk_000000_indices", "indptr": "tensor_000046_chunk_000000_indptr", "nnz": 1534, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000046_chunk_000001_indices", "indptr": "tensor_000046_chunk_000001_indptr", "nnz": 1536, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000046_chunk_000002_indices", "indptr": "tensor_000046_chunk_000002_indptr", "nnz": 1871, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000046_chunk_000003_indices", "indptr": "tensor_000046_chunk_000003_indptr", "nnz": 264, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000046_chunk_000004_indices", "indptr": "tensor_000046_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000046_chunk_000005_indices", "indptr": "tensor_000046_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000046_chunk_000006_indices", "indptr": "tensor_000046_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000046_chunk_000007_indices", "indptr": "tensor_000046_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000046_chunk_000008_indices", "indptr": "tensor_000046_chunk_000008_indptr", "nnz": 512, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000046_chunk_000009_indices", "indptr": "tensor_000046_chunk_000009_indptr", "nnz": 412, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000046_chunk_000010_indices", "indptr": "tensor_000046_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000046_chunk_000011_indices", "indptr": "tensor_000046_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000046_chunk_000012_indices", "indptr": "tensor_000046_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000046_chunk_000013_indices", "indptr": "tensor_000046_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 6810, "shape": [ 6830, 2048 ] }, "model.layers.11.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000133_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.11.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000045_chunk_000000_indices", "indptr": "tensor_000045_chunk_000000_indptr", "nnz": 223, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000045_chunk_000001_indices", "indptr": "tensor_000045_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000045_chunk_000002_indices", "indptr": "tensor_000045_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000045_chunk_000003_indices", "indptr": "tensor_000045_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 223, "shape": [ 2048, 2048 ] }, "model.layers.11.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000044_chunk_000000_indices", "indptr": "tensor_000044_chunk_000000_indptr", "nnz": 194, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000044_chunk_000001_indices", "indptr": "tensor_000044_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000044_chunk_000002_indices", "indptr": "tensor_000044_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000044_chunk_000003_indices", "indptr": "tensor_000044_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000044_chunk_000004_indices", "indptr": "tensor_000044_chunk_000004_indptr", "nnz": 194, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000044_chunk_000005_indices", "indptr": "tensor_000044_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000044_chunk_000006_indices", "indptr": "tensor_000044_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000044_chunk_000007_indices", "indptr": "tensor_000044_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000044_chunk_000008_indices", "indptr": "tensor_000044_chunk_000008_indptr", "nnz": 223, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000044_chunk_000009_indices", "indptr": "tensor_000044_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000044_chunk_000010_indices", "indptr": "tensor_000044_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000044_chunk_000011_indices", "indptr": "tensor_000044_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 611, "shape": [ 6144, 2048 ] }, "model.layers.12.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000134_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.12.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000051_chunk_000000_indices", "indptr": "tensor_000051_chunk_000000_indptr", "nnz": 578, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000051_chunk_000001_indices", "indptr": "tensor_000051_chunk_000001_indptr", "nnz": 156, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000051_chunk_000002_indices", "indptr": "tensor_000051_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000051_chunk_000003_indices", "indptr": "tensor_000051_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000051_chunk_000004_indices", "indptr": "tensor_000051_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000051_chunk_000005_indices", "indptr": "tensor_000051_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000051_chunk_000006_indices", "indptr": "tensor_000051_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 734, "shape": [ 2048, 3415 ] }, "model.layers.12.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000050_chunk_000000_indices", "indptr": "tensor_000050_chunk_000000_indptr", "nnz": 1770, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000050_chunk_000001_indices", "indptr": "tensor_000050_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000050_chunk_000002_indices", "indptr": "tensor_000050_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000050_chunk_000003_indices", "indptr": "tensor_000050_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000050_chunk_000004_indices", "indptr": "tensor_000050_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000050_chunk_000005_indices", "indptr": "tensor_000050_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000050_chunk_000006_indices", "indptr": "tensor_000050_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000050_chunk_000007_indices", "indptr": "tensor_000050_chunk_000007_indptr", "nnz": 332, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000050_chunk_000008_indices", "indptr": "tensor_000050_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000050_chunk_000009_indices", "indptr": "tensor_000050_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000050_chunk_000010_indices", "indptr": "tensor_000050_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000050_chunk_000011_indices", "indptr": "tensor_000050_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000050_chunk_000012_indices", "indptr": "tensor_000050_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000050_chunk_000013_indices", "indptr": "tensor_000050_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2271, "shape": [ 6830, 2048 ] }, "model.layers.12.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000135_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.12.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000049_chunk_000000_indices", "indptr": "tensor_000049_chunk_000000_indptr", "nnz": 4, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000049_chunk_000001_indices", "indptr": "tensor_000049_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000049_chunk_000002_indices", "indptr": "tensor_000049_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000049_chunk_000003_indices", "indptr": "tensor_000049_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 4, "shape": [ 2048, 2048 ] }, "model.layers.12.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000048_chunk_000000_indices", "indptr": "tensor_000048_chunk_000000_indptr", "nnz": 128, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000048_chunk_000001_indices", "indptr": "tensor_000048_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000048_chunk_000002_indices", "indptr": "tensor_000048_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000048_chunk_000003_indices", "indptr": "tensor_000048_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000048_chunk_000004_indices", "indptr": "tensor_000048_chunk_000004_indptr", "nnz": 128, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000048_chunk_000005_indices", "indptr": "tensor_000048_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000048_chunk_000006_indices", "indptr": "tensor_000048_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000048_chunk_000007_indices", "indptr": "tensor_000048_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000048_chunk_000008_indices", "indptr": "tensor_000048_chunk_000008_indptr", "nnz": 8, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000048_chunk_000009_indices", "indptr": "tensor_000048_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000048_chunk_000010_indices", "indptr": "tensor_000048_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000048_chunk_000011_indices", "indptr": "tensor_000048_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 264, "shape": [ 6144, 2048 ] }, "model.layers.13.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000136_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.13.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000055_chunk_000000_indices", "indptr": "tensor_000055_chunk_000000_indptr", "nnz": 643, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000055_chunk_000001_indices", "indptr": "tensor_000055_chunk_000001_indptr", "nnz": 366, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000055_chunk_000002_indices", "indptr": "tensor_000055_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000055_chunk_000003_indices", "indptr": "tensor_000055_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000055_chunk_000004_indices", "indptr": "tensor_000055_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000055_chunk_000005_indices", "indptr": "tensor_000055_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000055_chunk_000006_indices", "indptr": "tensor_000055_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1009, "shape": [ 2048, 3415 ] }, "model.layers.13.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000054_chunk_000000_indices", "indptr": "tensor_000054_chunk_000000_indptr", "nnz": 1823, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000054_chunk_000001_indices", "indptr": "tensor_000054_chunk_000001_indptr", "nnz": 144, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000054_chunk_000002_indices", "indptr": "tensor_000054_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000054_chunk_000003_indices", "indptr": "tensor_000054_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000054_chunk_000004_indices", "indptr": "tensor_000054_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000054_chunk_000005_indices", "indptr": "tensor_000054_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000054_chunk_000006_indices", "indptr": "tensor_000054_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000054_chunk_000007_indices", "indptr": "tensor_000054_chunk_000007_indptr", "nnz": 474, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000054_chunk_000008_indices", "indptr": "tensor_000054_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000054_chunk_000009_indices", "indptr": "tensor_000054_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000054_chunk_000010_indices", "indptr": "tensor_000054_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000054_chunk_000011_indices", "indptr": "tensor_000054_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000054_chunk_000012_indices", "indptr": "tensor_000054_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000054_chunk_000013_indices", "indptr": "tensor_000054_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2610, "shape": [ 6830, 2048 ] }, "model.layers.13.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000137_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.13.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000053_chunk_000000_indices", "indptr": "tensor_000053_chunk_000000_indptr", "nnz": 10, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000053_chunk_000001_indices", "indptr": "tensor_000053_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000053_chunk_000002_indices", "indptr": "tensor_000053_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000053_chunk_000003_indices", "indptr": "tensor_000053_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 10, "shape": [ 2048, 2048 ] }, "model.layers.13.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000052_chunk_000000_indices", "indptr": "tensor_000052_chunk_000000_indptr", "nnz": 192, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000052_chunk_000001_indices", "indptr": "tensor_000052_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000052_chunk_000002_indices", "indptr": "tensor_000052_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000052_chunk_000003_indices", "indptr": "tensor_000052_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000052_chunk_000004_indices", "indptr": "tensor_000052_chunk_000004_indptr", "nnz": 192, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000052_chunk_000005_indices", "indptr": "tensor_000052_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000052_chunk_000006_indices", "indptr": "tensor_000052_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000052_chunk_000007_indices", "indptr": "tensor_000052_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000052_chunk_000008_indices", "indptr": "tensor_000052_chunk_000008_indptr", "nnz": 10, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000052_chunk_000009_indices", "indptr": "tensor_000052_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000052_chunk_000010_indices", "indptr": "tensor_000052_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000052_chunk_000011_indices", "indptr": "tensor_000052_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 394, "shape": [ 6144, 2048 ] }, "model.layers.14.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000138_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.14.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000059_chunk_000000_indices", "indptr": "tensor_000059_chunk_000000_indptr", "nnz": 278, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000059_chunk_000001_indices", "indptr": "tensor_000059_chunk_000001_indptr", "nnz": 1191, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000059_chunk_000002_indices", "indptr": "tensor_000059_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000059_chunk_000003_indices", "indptr": "tensor_000059_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000059_chunk_000004_indices", "indptr": "tensor_000059_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000059_chunk_000005_indices", "indptr": "tensor_000059_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000059_chunk_000006_indices", "indptr": "tensor_000059_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1469, "shape": [ 2048, 3415 ] }, "model.layers.14.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000058_chunk_000000_indices", "indptr": "tensor_000058_chunk_000000_indptr", "nnz": 1199, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000058_chunk_000001_indices", "indptr": "tensor_000058_chunk_000001_indptr", "nnz": 522, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000058_chunk_000002_indices", "indptr": "tensor_000058_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000058_chunk_000003_indices", "indptr": "tensor_000058_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000058_chunk_000004_indices", "indptr": "tensor_000058_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000058_chunk_000005_indices", "indptr": "tensor_000058_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000058_chunk_000006_indices", "indptr": "tensor_000058_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000058_chunk_000007_indices", "indptr": "tensor_000058_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000058_chunk_000008_indices", "indptr": "tensor_000058_chunk_000008_indptr", "nnz": 241, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000058_chunk_000009_indices", "indptr": "tensor_000058_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000058_chunk_000010_indices", "indptr": "tensor_000058_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000058_chunk_000011_indices", "indptr": "tensor_000058_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000058_chunk_000012_indices", "indptr": "tensor_000058_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000058_chunk_000013_indices", "indptr": "tensor_000058_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2643, "shape": [ 6830, 2048 ] }, "model.layers.14.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000139_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.14.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000057_chunk_000000_indices", "indptr": "tensor_000057_chunk_000000_indptr", "nnz": 3, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000057_chunk_000001_indices", "indptr": "tensor_000057_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000057_chunk_000002_indices", "indptr": "tensor_000057_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000057_chunk_000003_indices", "indptr": "tensor_000057_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 3, "shape": [ 2048, 2048 ] }, "model.layers.14.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000056_chunk_000000_indices", "indptr": "tensor_000056_chunk_000000_indptr", "nnz": 128, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000056_chunk_000001_indices", "indptr": "tensor_000056_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000056_chunk_000002_indices", "indptr": "tensor_000056_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000056_chunk_000003_indices", "indptr": "tensor_000056_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000056_chunk_000004_indices", "indptr": "tensor_000056_chunk_000004_indptr", "nnz": 128, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000056_chunk_000005_indices", "indptr": "tensor_000056_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000056_chunk_000006_indices", "indptr": "tensor_000056_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000056_chunk_000007_indices", "indptr": "tensor_000056_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000056_chunk_000008_indices", "indptr": "tensor_000056_chunk_000008_indptr", "nnz": 3, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000056_chunk_000009_indices", "indptr": "tensor_000056_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000056_chunk_000010_indices", "indptr": "tensor_000056_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000056_chunk_000011_indices", "indptr": "tensor_000056_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 259, "shape": [ 6144, 2048 ] }, "model.layers.15.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000140_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.15.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000063_chunk_000000_indices", "indptr": "tensor_000063_chunk_000000_indptr", "nnz": 1236, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000063_chunk_000001_indices", "indptr": "tensor_000063_chunk_000001_indptr", "nnz": 108, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000063_chunk_000002_indices", "indptr": "tensor_000063_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000063_chunk_000003_indices", "indptr": "tensor_000063_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000063_chunk_000004_indices", "indptr": "tensor_000063_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000063_chunk_000005_indices", "indptr": "tensor_000063_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000063_chunk_000006_indices", "indptr": "tensor_000063_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1344, "shape": [ 2048, 3415 ] }, "model.layers.15.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000062_chunk_000000_indices", "indptr": "tensor_000062_chunk_000000_indptr", "nnz": 1507, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000062_chunk_000001_indices", "indptr": "tensor_000062_chunk_000001_indptr", "nnz": 712, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000062_chunk_000002_indices", "indptr": "tensor_000062_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000062_chunk_000003_indices", "indptr": "tensor_000062_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000062_chunk_000004_indices", "indptr": "tensor_000062_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000062_chunk_000005_indices", "indptr": "tensor_000062_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000062_chunk_000006_indices", "indptr": "tensor_000062_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000062_chunk_000007_indices", "indptr": "tensor_000062_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000062_chunk_000008_indices", "indptr": "tensor_000062_chunk_000008_indptr", "nnz": 116, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000062_chunk_000009_indices", "indptr": "tensor_000062_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000062_chunk_000010_indices", "indptr": "tensor_000062_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000062_chunk_000011_indices", "indptr": "tensor_000062_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000062_chunk_000012_indices", "indptr": "tensor_000062_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000062_chunk_000013_indices", "indptr": "tensor_000062_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 3016, "shape": [ 6830, 2048 ] }, "model.layers.15.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000141_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.15.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000061_chunk_000000_indices", "indptr": "tensor_000061_chunk_000000_indptr", "nnz": 3, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000061_chunk_000001_indices", "indptr": "tensor_000061_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000061_chunk_000002_indices", "indptr": "tensor_000061_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000061_chunk_000003_indices", "indptr": "tensor_000061_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 3, "shape": [ 2048, 2048 ] }, "model.layers.15.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000060_chunk_000000_indices", "indptr": "tensor_000060_chunk_000000_indptr", "nnz": 128, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000060_chunk_000001_indices", "indptr": "tensor_000060_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000060_chunk_000002_indices", "indptr": "tensor_000060_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000060_chunk_000003_indices", "indptr": "tensor_000060_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000060_chunk_000004_indices", "indptr": "tensor_000060_chunk_000004_indptr", "nnz": 128, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000060_chunk_000005_indices", "indptr": "tensor_000060_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000060_chunk_000006_indices", "indptr": "tensor_000060_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000060_chunk_000007_indices", "indptr": "tensor_000060_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000060_chunk_000008_indices", "indptr": "tensor_000060_chunk_000008_indptr", "nnz": 3, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000060_chunk_000009_indices", "indptr": "tensor_000060_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000060_chunk_000010_indices", "indptr": "tensor_000060_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000060_chunk_000011_indices", "indptr": "tensor_000060_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 259, "shape": [ 6144, 2048 ] }, "model.layers.16.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000142_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.16.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000067_chunk_000000_indices", "indptr": "tensor_000067_chunk_000000_indptr", "nnz": 1085, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000067_chunk_000001_indices", "indptr": "tensor_000067_chunk_000001_indptr", "nnz": 106, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000067_chunk_000002_indices", "indptr": "tensor_000067_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000067_chunk_000003_indices", "indptr": "tensor_000067_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000067_chunk_000004_indices", "indptr": "tensor_000067_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000067_chunk_000005_indices", "indptr": "tensor_000067_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000067_chunk_000006_indices", "indptr": "tensor_000067_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1191, "shape": [ 2048, 3415 ] }, "model.layers.16.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000066_chunk_000000_indices", "indptr": "tensor_000066_chunk_000000_indptr", "nnz": 1753, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000066_chunk_000001_indices", "indptr": "tensor_000066_chunk_000001_indptr", "nnz": 382, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000066_chunk_000002_indices", "indptr": "tensor_000066_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000066_chunk_000003_indices", "indptr": "tensor_000066_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000066_chunk_000004_indices", "indptr": "tensor_000066_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000066_chunk_000005_indices", "indptr": "tensor_000066_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000066_chunk_000006_indices", "indptr": "tensor_000066_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000066_chunk_000007_indices", "indptr": "tensor_000066_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000066_chunk_000008_indices", "indptr": "tensor_000066_chunk_000008_indptr", "nnz": 108, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000066_chunk_000009_indices", "indptr": "tensor_000066_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000066_chunk_000010_indices", "indptr": "tensor_000066_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000066_chunk_000011_indices", "indptr": "tensor_000066_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000066_chunk_000012_indices", "indptr": "tensor_000066_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000066_chunk_000013_indices", "indptr": "tensor_000066_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2924, "shape": [ 6830, 2048 ] }, "model.layers.16.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000143_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.16.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000065_chunk_000000_indices", "indptr": "tensor_000065_chunk_000000_indptr", "nnz": 3, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000065_chunk_000001_indices", "indptr": "tensor_000065_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000065_chunk_000002_indices", "indptr": "tensor_000065_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000065_chunk_000003_indices", "indptr": "tensor_000065_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 3, "shape": [ 2048, 2048 ] }, "model.layers.16.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000064_chunk_000000_indices", "indptr": "tensor_000064_chunk_000000_indptr", "nnz": 128, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000064_chunk_000001_indices", "indptr": "tensor_000064_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000064_chunk_000002_indices", "indptr": "tensor_000064_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000064_chunk_000003_indices", "indptr": "tensor_000064_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000064_chunk_000004_indices", "indptr": "tensor_000064_chunk_000004_indptr", "nnz": 128, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000064_chunk_000005_indices", "indptr": "tensor_000064_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000064_chunk_000006_indices", "indptr": "tensor_000064_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000064_chunk_000007_indices", "indptr": "tensor_000064_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000064_chunk_000008_indices", "indptr": "tensor_000064_chunk_000008_indptr", "nnz": 3, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000064_chunk_000009_indices", "indptr": "tensor_000064_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000064_chunk_000010_indices", "indptr": "tensor_000064_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000064_chunk_000011_indices", "indptr": "tensor_000064_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 259, "shape": [ 6144, 2048 ] }, "model.layers.17.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000144_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.17.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000071_chunk_000000_indices", "indptr": "tensor_000071_chunk_000000_indptr", "nnz": 702, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000071_chunk_000001_indices", "indptr": "tensor_000071_chunk_000001_indptr", "nnz": 252, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000071_chunk_000002_indices", "indptr": "tensor_000071_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000071_chunk_000003_indices", "indptr": "tensor_000071_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000071_chunk_000004_indices", "indptr": "tensor_000071_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000071_chunk_000005_indices", "indptr": "tensor_000071_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000071_chunk_000006_indices", "indptr": "tensor_000071_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 954, "shape": [ 2048, 3415 ] }, "model.layers.17.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000070_chunk_000000_indices", "indptr": "tensor_000070_chunk_000000_indptr", "nnz": 1242, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000070_chunk_000001_indices", "indptr": "tensor_000070_chunk_000001_indptr", "nnz": 164, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000070_chunk_000002_indices", "indptr": "tensor_000070_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000070_chunk_000003_indices", "indptr": "tensor_000070_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000070_chunk_000004_indices", "indptr": "tensor_000070_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000070_chunk_000005_indices", "indptr": "tensor_000070_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000070_chunk_000006_indices", "indptr": "tensor_000070_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000070_chunk_000007_indices", "indptr": "tensor_000070_chunk_000007_indptr", "nnz": 507, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000070_chunk_000008_indices", "indptr": "tensor_000070_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000070_chunk_000009_indices", "indptr": "tensor_000070_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000070_chunk_000010_indices", "indptr": "tensor_000070_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000070_chunk_000011_indices", "indptr": "tensor_000070_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000070_chunk_000012_indices", "indptr": "tensor_000070_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000070_chunk_000013_indices", "indptr": "tensor_000070_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2082, "shape": [ 6830, 2048 ] }, "model.layers.17.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000145_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.17.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000069_chunk_000000_indices", "indptr": "tensor_000069_chunk_000000_indptr", "nnz": 72, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000069_chunk_000001_indices", "indptr": "tensor_000069_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000069_chunk_000002_indices", "indptr": "tensor_000069_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000069_chunk_000003_indices", "indptr": "tensor_000069_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 72, "shape": [ 2048, 2048 ] }, "model.layers.17.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000068_chunk_000000_indices", "indptr": "tensor_000068_chunk_000000_indptr", "nnz": 192, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000068_chunk_000001_indices", "indptr": "tensor_000068_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000068_chunk_000002_indices", "indptr": "tensor_000068_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000068_chunk_000003_indices", "indptr": "tensor_000068_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000068_chunk_000004_indices", "indptr": "tensor_000068_chunk_000004_indptr", "nnz": 192, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000068_chunk_000005_indices", "indptr": "tensor_000068_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000068_chunk_000006_indices", "indptr": "tensor_000068_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000068_chunk_000007_indices", "indptr": "tensor_000068_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000068_chunk_000008_indices", "indptr": "tensor_000068_chunk_000008_indptr", "nnz": 72, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000068_chunk_000009_indices", "indptr": "tensor_000068_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000068_chunk_000010_indices", "indptr": "tensor_000068_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000068_chunk_000011_indices", "indptr": "tensor_000068_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 456, "shape": [ 6144, 2048 ] }, "model.layers.18.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000146_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.18.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000075_chunk_000000_indices", "indptr": "tensor_000075_chunk_000000_indptr", "nnz": 158, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000075_chunk_000001_indices", "indptr": "tensor_000075_chunk_000001_indptr", "nnz": 24, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000075_chunk_000002_indices", "indptr": "tensor_000075_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000075_chunk_000003_indices", "indptr": "tensor_000075_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000075_chunk_000004_indices", "indptr": "tensor_000075_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000075_chunk_000005_indices", "indptr": "tensor_000075_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000075_chunk_000006_indices", "indptr": "tensor_000075_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 182, "shape": [ 2048, 3415 ] }, "model.layers.18.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000074_chunk_000000_indices", "indptr": "tensor_000074_chunk_000000_indptr", "nnz": 180, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000074_chunk_000001_indices", "indptr": "tensor_000074_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000074_chunk_000002_indices", "indptr": "tensor_000074_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000074_chunk_000003_indices", "indptr": "tensor_000074_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000074_chunk_000004_indices", "indptr": "tensor_000074_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000074_chunk_000005_indices", "indptr": "tensor_000074_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000074_chunk_000006_indices", "indptr": "tensor_000074_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000074_chunk_000007_indices", "indptr": "tensor_000074_chunk_000007_indptr", "nnz": 11, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000074_chunk_000008_indices", "indptr": "tensor_000074_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000074_chunk_000009_indices", "indptr": "tensor_000074_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000074_chunk_000010_indices", "indptr": "tensor_000074_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000074_chunk_000011_indices", "indptr": "tensor_000074_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000074_chunk_000012_indices", "indptr": "tensor_000074_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000074_chunk_000013_indices", "indptr": "tensor_000074_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 360, "shape": [ 6830, 2048 ] }, "model.layers.18.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000147_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.18.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000073_chunk_000000_indices", "indptr": "tensor_000073_chunk_000000_indptr", "nnz": 172, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000073_chunk_000001_indices", "indptr": "tensor_000073_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000073_chunk_000002_indices", "indptr": "tensor_000073_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000073_chunk_000003_indices", "indptr": "tensor_000073_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 172, "shape": [ 2048, 2048 ] }, "model.layers.18.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000072_chunk_000000_indices", "indptr": "tensor_000072_chunk_000000_indptr", "nnz": 256, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000072_chunk_000001_indices", "indptr": "tensor_000072_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000072_chunk_000002_indices", "indptr": "tensor_000072_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000072_chunk_000003_indices", "indptr": "tensor_000072_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000072_chunk_000004_indices", "indptr": "tensor_000072_chunk_000004_indptr", "nnz": 256, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000072_chunk_000005_indices", "indptr": "tensor_000072_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000072_chunk_000006_indices", "indptr": "tensor_000072_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000072_chunk_000007_indices", "indptr": "tensor_000072_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000072_chunk_000008_indices", "indptr": "tensor_000072_chunk_000008_indptr", "nnz": 172, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000072_chunk_000009_indices", "indptr": "tensor_000072_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000072_chunk_000010_indices", "indptr": "tensor_000072_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000072_chunk_000011_indices", "indptr": "tensor_000072_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 684, "shape": [ 6144, 2048 ] }, "model.layers.19.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000148_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.19.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000079_chunk_000000_indices", "indptr": "tensor_000079_chunk_000000_indptr", "nnz": 76, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000079_chunk_000001_indices", "indptr": "tensor_000079_chunk_000001_indptr", "nnz": 8, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000079_chunk_000002_indices", "indptr": "tensor_000079_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000079_chunk_000003_indices", "indptr": "tensor_000079_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000079_chunk_000004_indices", "indptr": "tensor_000079_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000079_chunk_000005_indices", "indptr": "tensor_000079_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000079_chunk_000006_indices", "indptr": "tensor_000079_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 84, "shape": [ 2048, 3415 ] }, "model.layers.19.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000078_chunk_000000_indices", "indptr": "tensor_000078_chunk_000000_indptr", "nnz": 80, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000078_chunk_000001_indices", "indptr": "tensor_000078_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000078_chunk_000002_indices", "indptr": "tensor_000078_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000078_chunk_000003_indices", "indptr": "tensor_000078_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000078_chunk_000004_indices", "indptr": "tensor_000078_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000078_chunk_000005_indices", "indptr": "tensor_000078_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000078_chunk_000006_indices", "indptr": "tensor_000078_chunk_000006_indptr", "nnz": 80, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000078_chunk_000007_indices", "indptr": "tensor_000078_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000078_chunk_000008_indices", "indptr": "tensor_000078_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000078_chunk_000009_indices", "indptr": "tensor_000078_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000078_chunk_000010_indices", "indptr": "tensor_000078_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000078_chunk_000011_indices", "indptr": "tensor_000078_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000078_chunk_000012_indices", "indptr": "tensor_000078_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000078_chunk_000013_indices", "indptr": "tensor_000078_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 160, "shape": [ 6830, 2048 ] }, "model.layers.19.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000149_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.19.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000077_chunk_000000_indices", "indptr": "tensor_000077_chunk_000000_indptr", "nnz": 153, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000077_chunk_000001_indices", "indptr": "tensor_000077_chunk_000001_indptr", "nnz": 17, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000077_chunk_000002_indices", "indptr": "tensor_000077_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000077_chunk_000003_indices", "indptr": "tensor_000077_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 170, "shape": [ 2048, 2048 ] }, "model.layers.19.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000076_chunk_000000_indices", "indptr": "tensor_000076_chunk_000000_indptr", "nnz": 256, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000076_chunk_000001_indices", "indptr": "tensor_000076_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000076_chunk_000002_indices", "indptr": "tensor_000076_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000076_chunk_000003_indices", "indptr": "tensor_000076_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000076_chunk_000004_indices", "indptr": "tensor_000076_chunk_000004_indptr", "nnz": 256, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000076_chunk_000005_indices", "indptr": "tensor_000076_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000076_chunk_000006_indices", "indptr": "tensor_000076_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000076_chunk_000007_indices", "indptr": "tensor_000076_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000076_chunk_000008_indices", "indptr": "tensor_000076_chunk_000008_indptr", "nnz": 170, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000076_chunk_000009_indices", "indptr": "tensor_000076_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000076_chunk_000010_indices", "indptr": "tensor_000076_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000076_chunk_000011_indices", "indptr": "tensor_000076_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 682, "shape": [ 6144, 2048 ] }, "model.layers.2.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000114_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.2.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000011_chunk_000000_indices", "indptr": "tensor_000011_chunk_000000_indptr", "nnz": 75, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000011_chunk_000001_indices", "indptr": "tensor_000011_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000011_chunk_000002_indices", "indptr": "tensor_000011_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000011_chunk_000003_indices", "indptr": "tensor_000011_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000011_chunk_000004_indices", "indptr": "tensor_000011_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000011_chunk_000005_indices", "indptr": "tensor_000011_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000011_chunk_000006_indices", "indptr": "tensor_000011_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 75, "shape": [ 2048, 3415 ] }, "model.layers.2.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000010_chunk_000000_indices", "indptr": "tensor_000010_chunk_000000_indptr", "nnz": 143, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000010_chunk_000001_indices", "indptr": "tensor_000010_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000010_chunk_000002_indices", "indptr": "tensor_000010_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000010_chunk_000003_indices", "indptr": "tensor_000010_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000010_chunk_000004_indices", "indptr": "tensor_000010_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000010_chunk_000005_indices", "indptr": "tensor_000010_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000010_chunk_000006_indices", "indptr": "tensor_000010_chunk_000006_indptr", "nnz": 73, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000010_chunk_000007_indices", "indptr": "tensor_000010_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000010_chunk_000008_indices", "indptr": "tensor_000010_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000010_chunk_000009_indices", "indptr": "tensor_000010_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000010_chunk_000010_indices", "indptr": "tensor_000010_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000010_chunk_000011_indices", "indptr": "tensor_000010_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000010_chunk_000012_indices", "indptr": "tensor_000010_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000010_chunk_000013_indices", "indptr": "tensor_000010_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 216, "shape": [ 6830, 2048 ] }, "model.layers.2.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000115_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.2.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000009_chunk_000000_indices", "indptr": "tensor_000009_chunk_000000_indptr", "nnz": 8, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000009_chunk_000001_indices", "indptr": "tensor_000009_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000009_chunk_000002_indices", "indptr": "tensor_000009_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000009_chunk_000003_indices", "indptr": "tensor_000009_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 8, "shape": [ 2048, 2048 ] }, "model.layers.2.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000008_chunk_000000_indices", "indptr": "tensor_000008_chunk_000000_indptr", "nnz": 66, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000008_chunk_000001_indices", "indptr": "tensor_000008_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000008_chunk_000002_indices", "indptr": "tensor_000008_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000008_chunk_000003_indices", "indptr": "tensor_000008_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000008_chunk_000004_indices", "indptr": "tensor_000008_chunk_000004_indptr", "nnz": 66, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000008_chunk_000005_indices", "indptr": "tensor_000008_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000008_chunk_000006_indices", "indptr": "tensor_000008_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000008_chunk_000007_indices", "indptr": "tensor_000008_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000008_chunk_000008_indices", "indptr": "tensor_000008_chunk_000008_indptr", "nnz": 8, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000008_chunk_000009_indices", "indptr": "tensor_000008_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000008_chunk_000010_indices", "indptr": "tensor_000008_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000008_chunk_000011_indices", "indptr": "tensor_000008_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 140, "shape": [ 6144, 2048 ] }, "model.layers.20.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000150_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.20.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000083_chunk_000000_indices", "indptr": "tensor_000083_chunk_000000_indptr", "nnz": 28, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000083_chunk_000001_indices", "indptr": "tensor_000083_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000083_chunk_000002_indices", "indptr": "tensor_000083_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000083_chunk_000003_indices", "indptr": "tensor_000083_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000083_chunk_000004_indices", "indptr": "tensor_000083_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000083_chunk_000005_indices", "indptr": "tensor_000083_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000083_chunk_000006_indices", "indptr": "tensor_000083_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 28, "shape": [ 2048, 3415 ] }, "model.layers.20.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000082_chunk_000000_indices", "indptr": "tensor_000082_chunk_000000_indptr", "nnz": 55, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000082_chunk_000001_indices", "indptr": "tensor_000082_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000082_chunk_000002_indices", "indptr": "tensor_000082_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000082_chunk_000003_indices", "indptr": "tensor_000082_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000082_chunk_000004_indices", "indptr": "tensor_000082_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000082_chunk_000005_indices", "indptr": "tensor_000082_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000082_chunk_000006_indices", "indptr": "tensor_000082_chunk_000006_indptr", "nnz": 23, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000082_chunk_000007_indices", "indptr": "tensor_000082_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000082_chunk_000008_indices", "indptr": "tensor_000082_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000082_chunk_000009_indices", "indptr": "tensor_000082_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000082_chunk_000010_indices", "indptr": "tensor_000082_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000082_chunk_000011_indices", "indptr": "tensor_000082_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000082_chunk_000012_indices", "indptr": "tensor_000082_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000082_chunk_000013_indices", "indptr": "tensor_000082_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 78, "shape": [ 6830, 2048 ] }, "model.layers.20.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000151_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.20.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000081_chunk_000000_indices", "indptr": "tensor_000081_chunk_000000_indptr", "nnz": 2, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000081_chunk_000001_indices", "indptr": "tensor_000081_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000081_chunk_000002_indices", "indptr": "tensor_000081_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000081_chunk_000003_indices", "indptr": "tensor_000081_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2, "shape": [ 2048, 2048 ] }, "model.layers.20.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000080_chunk_000000_indices", "indptr": "tensor_000080_chunk_000000_indptr", "nnz": 128, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000080_chunk_000001_indices", "indptr": "tensor_000080_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000080_chunk_000002_indices", "indptr": "tensor_000080_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000080_chunk_000003_indices", "indptr": "tensor_000080_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000080_chunk_000004_indices", "indptr": "tensor_000080_chunk_000004_indptr", "nnz": 128, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000080_chunk_000005_indices", "indptr": "tensor_000080_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000080_chunk_000006_indices", "indptr": "tensor_000080_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000080_chunk_000007_indices", "indptr": "tensor_000080_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000080_chunk_000008_indices", "indptr": "tensor_000080_chunk_000008_indptr", "nnz": 2, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000080_chunk_000009_indices", "indptr": "tensor_000080_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000080_chunk_000010_indices", "indptr": "tensor_000080_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000080_chunk_000011_indices", "indptr": "tensor_000080_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 258, "shape": [ 6144, 2048 ] }, "model.layers.21.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000152_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.21.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000087_chunk_000000_indices", "indptr": "tensor_000087_chunk_000000_indptr", "nnz": 14, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000087_chunk_000001_indices", "indptr": "tensor_000087_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000087_chunk_000002_indices", "indptr": "tensor_000087_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000087_chunk_000003_indices", "indptr": "tensor_000087_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000087_chunk_000004_indices", "indptr": "tensor_000087_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000087_chunk_000005_indices", "indptr": "tensor_000087_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000087_chunk_000006_indices", "indptr": "tensor_000087_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 14, "shape": [ 2048, 3415 ] }, "model.layers.21.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000086_chunk_000000_indices", "indptr": "tensor_000086_chunk_000000_indptr", "nnz": 17, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000086_chunk_000001_indices", "indptr": "tensor_000086_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000086_chunk_000002_indices", "indptr": "tensor_000086_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000086_chunk_000003_indices", "indptr": "tensor_000086_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000086_chunk_000004_indices", "indptr": "tensor_000086_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000086_chunk_000005_indices", "indptr": "tensor_000086_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000086_chunk_000006_indices", "indptr": "tensor_000086_chunk_000006_indptr", "nnz": 13, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000086_chunk_000007_indices", "indptr": "tensor_000086_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000086_chunk_000008_indices", "indptr": "tensor_000086_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000086_chunk_000009_indices", "indptr": "tensor_000086_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000086_chunk_000010_indices", "indptr": "tensor_000086_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000086_chunk_000011_indices", "indptr": "tensor_000086_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000086_chunk_000012_indices", "indptr": "tensor_000086_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000086_chunk_000013_indices", "indptr": "tensor_000086_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 30, "shape": [ 6830, 2048 ] }, "model.layers.21.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000153_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.21.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000085_chunk_000000_indices", "indptr": "tensor_000085_chunk_000000_indptr", "nnz": 0, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000085_chunk_000001_indices", "indptr": "tensor_000085_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000085_chunk_000002_indices", "indptr": "tensor_000085_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000085_chunk_000003_indices", "indptr": "tensor_000085_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 0, "shape": [ 2048, 2048 ] }, "model.layers.21.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000084_chunk_000000_indices", "indptr": "tensor_000084_chunk_000000_indptr", "nnz": 0, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000084_chunk_000001_indices", "indptr": "tensor_000084_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000084_chunk_000002_indices", "indptr": "tensor_000084_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000084_chunk_000003_indices", "indptr": "tensor_000084_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000084_chunk_000004_indices", "indptr": "tensor_000084_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000084_chunk_000005_indices", "indptr": "tensor_000084_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000084_chunk_000006_indices", "indptr": "tensor_000084_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000084_chunk_000007_indices", "indptr": "tensor_000084_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000084_chunk_000008_indices", "indptr": "tensor_000084_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000084_chunk_000009_indices", "indptr": "tensor_000084_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000084_chunk_000010_indices", "indptr": "tensor_000084_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000084_chunk_000011_indices", "indptr": "tensor_000084_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 0, "shape": [ 6144, 2048 ] }, "model.layers.22.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000154_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.22.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000091_chunk_000000_indices", "indptr": "tensor_000091_chunk_000000_indptr", "nnz": 5, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000091_chunk_000001_indices", "indptr": "tensor_000091_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000091_chunk_000002_indices", "indptr": "tensor_000091_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000091_chunk_000003_indices", "indptr": "tensor_000091_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000091_chunk_000004_indices", "indptr": "tensor_000091_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000091_chunk_000005_indices", "indptr": "tensor_000091_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000091_chunk_000006_indices", "indptr": "tensor_000091_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 5, "shape": [ 2048, 3415 ] }, "model.layers.22.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000090_chunk_000000_indices", "indptr": "tensor_000090_chunk_000000_indptr", "nnz": 5, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000090_chunk_000001_indices", "indptr": "tensor_000090_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000090_chunk_000002_indices", "indptr": "tensor_000090_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000090_chunk_000003_indices", "indptr": "tensor_000090_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000090_chunk_000004_indices", "indptr": "tensor_000090_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000090_chunk_000005_indices", "indptr": "tensor_000090_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000090_chunk_000006_indices", "indptr": "tensor_000090_chunk_000006_indptr", "nnz": 5, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000090_chunk_000007_indices", "indptr": "tensor_000090_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000090_chunk_000008_indices", "indptr": "tensor_000090_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000090_chunk_000009_indices", "indptr": "tensor_000090_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000090_chunk_000010_indices", "indptr": "tensor_000090_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000090_chunk_000011_indices", "indptr": "tensor_000090_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000090_chunk_000012_indices", "indptr": "tensor_000090_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000090_chunk_000013_indices", "indptr": "tensor_000090_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 10, "shape": [ 6830, 2048 ] }, "model.layers.22.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000155_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.22.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000089_chunk_000000_indices", "indptr": "tensor_000089_chunk_000000_indptr", "nnz": 9, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000089_chunk_000001_indices", "indptr": "tensor_000089_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000089_chunk_000002_indices", "indptr": "tensor_000089_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000089_chunk_000003_indices", "indptr": "tensor_000089_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 9, "shape": [ 2048, 2048 ] }, "model.layers.22.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000088_chunk_000000_indices", "indptr": "tensor_000088_chunk_000000_indptr", "nnz": 128, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000088_chunk_000001_indices", "indptr": "tensor_000088_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000088_chunk_000002_indices", "indptr": "tensor_000088_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000088_chunk_000003_indices", "indptr": "tensor_000088_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000088_chunk_000004_indices", "indptr": "tensor_000088_chunk_000004_indptr", "nnz": 128, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000088_chunk_000005_indices", "indptr": "tensor_000088_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000088_chunk_000006_indices", "indptr": "tensor_000088_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000088_chunk_000007_indices", "indptr": "tensor_000088_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000088_chunk_000008_indices", "indptr": "tensor_000088_chunk_000008_indptr", "nnz": 9, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000088_chunk_000009_indices", "indptr": "tensor_000088_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000088_chunk_000010_indices", "indptr": "tensor_000088_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000088_chunk_000011_indices", "indptr": "tensor_000088_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 265, "shape": [ 6144, 2048 ] }, "model.layers.23.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000156_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.23.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000095_chunk_000000_indices", "indptr": "tensor_000095_chunk_000000_indptr", "nnz": 143, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000095_chunk_000001_indices", "indptr": "tensor_000095_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000095_chunk_000002_indices", "indptr": "tensor_000095_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000095_chunk_000003_indices", "indptr": "tensor_000095_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000095_chunk_000004_indices", "indptr": "tensor_000095_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000095_chunk_000005_indices", "indptr": "tensor_000095_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000095_chunk_000006_indices", "indptr": "tensor_000095_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 143, "shape": [ 2048, 3415 ] }, "model.layers.23.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000094_chunk_000000_indices", "indptr": "tensor_000094_chunk_000000_indptr", "nnz": 93, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000094_chunk_000001_indices", "indptr": "tensor_000094_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000094_chunk_000002_indices", "indptr": "tensor_000094_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000094_chunk_000003_indices", "indptr": "tensor_000094_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000094_chunk_000004_indices", "indptr": "tensor_000094_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000094_chunk_000005_indices", "indptr": "tensor_000094_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000094_chunk_000006_indices", "indptr": "tensor_000094_chunk_000006_indptr", "nnz": 47, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000094_chunk_000007_indices", "indptr": "tensor_000094_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000094_chunk_000008_indices", "indptr": "tensor_000094_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000094_chunk_000009_indices", "indptr": "tensor_000094_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000094_chunk_000010_indices", "indptr": "tensor_000094_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000094_chunk_000011_indices", "indptr": "tensor_000094_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000094_chunk_000012_indices", "indptr": "tensor_000094_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000094_chunk_000013_indices", "indptr": "tensor_000094_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 140, "shape": [ 6830, 2048 ] }, "model.layers.23.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000157_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.23.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000093_chunk_000000_indices", "indptr": "tensor_000093_chunk_000000_indptr", "nnz": 1, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000093_chunk_000001_indices", "indptr": "tensor_000093_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000093_chunk_000002_indices", "indptr": "tensor_000093_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000093_chunk_000003_indices", "indptr": "tensor_000093_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1, "shape": [ 2048, 2048 ] }, "model.layers.23.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000092_chunk_000000_indices", "indptr": "tensor_000092_chunk_000000_indptr", "nnz": 64, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000092_chunk_000001_indices", "indptr": "tensor_000092_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000092_chunk_000002_indices", "indptr": "tensor_000092_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000092_chunk_000003_indices", "indptr": "tensor_000092_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000092_chunk_000004_indices", "indptr": "tensor_000092_chunk_000004_indptr", "nnz": 64, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000092_chunk_000005_indices", "indptr": "tensor_000092_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000092_chunk_000006_indices", "indptr": "tensor_000092_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000092_chunk_000007_indices", "indptr": "tensor_000092_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000092_chunk_000008_indices", "indptr": "tensor_000092_chunk_000008_indptr", "nnz": 1, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000092_chunk_000009_indices", "indptr": "tensor_000092_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000092_chunk_000010_indices", "indptr": "tensor_000092_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000092_chunk_000011_indices", "indptr": "tensor_000092_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 129, "shape": [ 6144, 2048 ] }, "model.layers.24.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000158_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.24.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000099_chunk_000000_indices", "indptr": "tensor_000099_chunk_000000_indptr", "nnz": 392, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000099_chunk_000001_indices", "indptr": "tensor_000099_chunk_000001_indptr", "nnz": 289, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000099_chunk_000002_indices", "indptr": "tensor_000099_chunk_000002_indptr", "nnz": 307, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000099_chunk_000003_indices", "indptr": "tensor_000099_chunk_000003_indptr", "nnz": 307, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000099_chunk_000004_indices", "indptr": "tensor_000099_chunk_000004_indptr", "nnz": 307, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000099_chunk_000005_indices", "indptr": "tensor_000099_chunk_000005_indptr", "nnz": 307, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000099_chunk_000006_indices", "indptr": "tensor_000099_chunk_000006_indptr", "nnz": 53, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1962, "shape": [ 2048, 3415 ] }, "model.layers.24.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000098_chunk_000000_indices", "indptr": "tensor_000098_chunk_000000_indptr", "nnz": 511, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000098_chunk_000001_indices", "indptr": "tensor_000098_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000098_chunk_000002_indices", "indptr": "tensor_000098_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000098_chunk_000003_indices", "indptr": "tensor_000098_chunk_000003_indptr", "nnz": 427, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000098_chunk_000004_indices", "indptr": "tensor_000098_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000098_chunk_000005_indices", "indptr": "tensor_000098_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000098_chunk_000006_indices", "indptr": "tensor_000098_chunk_000006_indptr", "nnz": 168, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000098_chunk_000007_indices", "indptr": "tensor_000098_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000098_chunk_000008_indices", "indptr": "tensor_000098_chunk_000008_indptr", "nnz": 512, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000098_chunk_000009_indices", "indptr": "tensor_000098_chunk_000009_indptr", "nnz": 512, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000098_chunk_000010_indices", "indptr": "tensor_000098_chunk_000010_indptr", "nnz": 258, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000098_chunk_000011_indices", "indptr": "tensor_000098_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000098_chunk_000012_indices", "indptr": "tensor_000098_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000098_chunk_000013_indices", "indptr": "tensor_000098_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 3924, "shape": [ 6830, 2048 ] }, "model.layers.24.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000159_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.24.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000097_chunk_000000_indices", "indptr": "tensor_000097_chunk_000000_indptr", "nnz": 1, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000097_chunk_000001_indices", "indptr": "tensor_000097_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000097_chunk_000002_indices", "indptr": "tensor_000097_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000097_chunk_000003_indices", "indptr": "tensor_000097_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1, "shape": [ 2048, 2048 ] }, "model.layers.24.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000096_chunk_000000_indices", "indptr": "tensor_000096_chunk_000000_indptr", "nnz": 64, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000096_chunk_000001_indices", "indptr": "tensor_000096_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000096_chunk_000002_indices", "indptr": "tensor_000096_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000096_chunk_000003_indices", "indptr": "tensor_000096_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000096_chunk_000004_indices", "indptr": "tensor_000096_chunk_000004_indptr", "nnz": 64, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000096_chunk_000005_indices", "indptr": "tensor_000096_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000096_chunk_000006_indices", "indptr": "tensor_000096_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000096_chunk_000007_indices", "indptr": "tensor_000096_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000096_chunk_000008_indices", "indptr": "tensor_000096_chunk_000008_indptr", "nnz": 1, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000096_chunk_000009_indices", "indptr": "tensor_000096_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000096_chunk_000010_indices", "indptr": "tensor_000096_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000096_chunk_000011_indices", "indptr": "tensor_000096_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 129, "shape": [ 6144, 2048 ] }, "model.layers.25.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000160_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.25.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000103_chunk_000000_indices", "indptr": "tensor_000103_chunk_000000_indptr", "nnz": 253, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000103_chunk_000001_indices", "indptr": "tensor_000103_chunk_000001_indptr", "nnz": 492, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000103_chunk_000002_indices", "indptr": "tensor_000103_chunk_000002_indptr", "nnz": 614, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000103_chunk_000003_indices", "indptr": "tensor_000103_chunk_000003_indptr", "nnz": 614, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000103_chunk_000004_indices", "indptr": "tensor_000103_chunk_000004_indptr", "nnz": 614, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000103_chunk_000005_indices", "indptr": "tensor_000103_chunk_000005_indptr", "nnz": 614, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000103_chunk_000006_indices", "indptr": "tensor_000103_chunk_000006_indptr", "nnz": 213, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 3414, "shape": [ 2048, 3415 ] }, "model.layers.25.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000102_chunk_000000_indices", "indptr": "tensor_000102_chunk_000000_indptr", "nnz": 511, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000102_chunk_000001_indices", "indptr": "tensor_000102_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000102_chunk_000002_indices", "indptr": "tensor_000102_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000102_chunk_000003_indices", "indptr": "tensor_000102_chunk_000003_indptr", "nnz": 512, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000102_chunk_000004_indices", "indptr": "tensor_000102_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000102_chunk_000005_indices", "indptr": "tensor_000102_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000102_chunk_000006_indices", "indptr": "tensor_000102_chunk_000006_indptr", "nnz": 511, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000102_chunk_000007_indices", "indptr": "tensor_000102_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000102_chunk_000008_indices", "indptr": "tensor_000102_chunk_000008_indptr", "nnz": 512, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000102_chunk_000009_indices", "indptr": "tensor_000102_chunk_000009_indptr", "nnz": 512, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000102_chunk_000010_indices", "indptr": "tensor_000102_chunk_000010_indptr", "nnz": 512, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000102_chunk_000011_indices", "indptr": "tensor_000102_chunk_000011_indptr", "nnz": 512, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000102_chunk_000012_indices", "indptr": "tensor_000102_chunk_000012_indptr", "nnz": 512, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000102_chunk_000013_indices", "indptr": "tensor_000102_chunk_000013_indptr", "nnz": 174, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 6828, "shape": [ 6830, 2048 ] }, "model.layers.25.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000161_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.25.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000101_chunk_000000_indices", "indptr": "tensor_000101_chunk_000000_indptr", "nnz": 1632, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000101_chunk_000001_indices", "indptr": "tensor_000101_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000101_chunk_000002_indices", "indptr": "tensor_000101_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000101_chunk_000003_indices", "indptr": "tensor_000101_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1632, "shape": [ 2048, 2048 ] }, "model.layers.25.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000100_chunk_000000_indices", "indptr": "tensor_000100_chunk_000000_indptr", "nnz": 512, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000100_chunk_000001_indices", "indptr": "tensor_000100_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000100_chunk_000002_indices", "indptr": "tensor_000100_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000100_chunk_000003_indices", "indptr": "tensor_000100_chunk_000003_indptr", "nnz": 512, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000100_chunk_000004_indices", "indptr": "tensor_000100_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000100_chunk_000005_indices", "indptr": "tensor_000100_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000100_chunk_000006_indices", "indptr": "tensor_000100_chunk_000006_indptr", "nnz": 512, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000100_chunk_000007_indices", "indptr": "tensor_000100_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000100_chunk_000008_indices", "indptr": "tensor_000100_chunk_000008_indptr", "nnz": 408, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000100_chunk_000009_indices", "indptr": "tensor_000100_chunk_000009_indptr", "nnz": 408, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000100_chunk_000010_indices", "indptr": "tensor_000100_chunk_000010_indptr", "nnz": 408, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000100_chunk_000011_indices", "indptr": "tensor_000100_chunk_000011_indptr", "nnz": 408, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 5728, "shape": [ 6144, 2048 ] }, "model.layers.26.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000162_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.26.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000107_chunk_000000_indices", "indptr": "tensor_000107_chunk_000000_indptr", "nnz": 229, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000107_chunk_000001_indices", "indptr": "tensor_000107_chunk_000001_indptr", "nnz": 46, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000107_chunk_000002_indices", "indptr": "tensor_000107_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000107_chunk_000003_indices", "indptr": "tensor_000107_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000107_chunk_000004_indices", "indptr": "tensor_000107_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000107_chunk_000005_indices", "indptr": "tensor_000107_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000107_chunk_000006_indices", "indptr": "tensor_000107_chunk_000006_indptr", "nnz": 204, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 479, "shape": [ 2048, 3415 ] }, "model.layers.26.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000106_chunk_000000_indices", "indptr": "tensor_000106_chunk_000000_indptr", "nnz": 479, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000106_chunk_000001_indices", "indptr": "tensor_000106_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000106_chunk_000002_indices", "indptr": "tensor_000106_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000106_chunk_000003_indices", "indptr": "tensor_000106_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000106_chunk_000004_indices", "indptr": "tensor_000106_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000106_chunk_000005_indices", "indptr": "tensor_000106_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000106_chunk_000006_indices", "indptr": "tensor_000106_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000106_chunk_000007_indices", "indptr": "tensor_000106_chunk_000007_indptr", "nnz": 310, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000106_chunk_000008_indices", "indptr": "tensor_000106_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000106_chunk_000009_indices", "indptr": "tensor_000106_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000106_chunk_000010_indices", "indptr": "tensor_000106_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000106_chunk_000011_indices", "indptr": "tensor_000106_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000106_chunk_000012_indices", "indptr": "tensor_000106_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000106_chunk_000013_indices", "indptr": "tensor_000106_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 958, "shape": [ 6830, 2048 ] }, "model.layers.26.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000163_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.26.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000105_chunk_000000_indices", "indptr": "tensor_000105_chunk_000000_indptr", "nnz": 88, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000105_chunk_000001_indices", "indptr": "tensor_000105_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000105_chunk_000002_indices", "indptr": "tensor_000105_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000105_chunk_000003_indices", "indptr": "tensor_000105_chunk_000003_indptr", "nnz": 117, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 205, "shape": [ 2048, 2048 ] }, "model.layers.26.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000104_chunk_000000_indices", "indptr": "tensor_000104_chunk_000000_indptr", "nnz": 320, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000104_chunk_000001_indices", "indptr": "tensor_000104_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000104_chunk_000002_indices", "indptr": "tensor_000104_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000104_chunk_000003_indices", "indptr": "tensor_000104_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000104_chunk_000004_indices", "indptr": "tensor_000104_chunk_000004_indptr", "nnz": 320, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000104_chunk_000005_indices", "indptr": "tensor_000104_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000104_chunk_000006_indices", "indptr": "tensor_000104_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000104_chunk_000007_indices", "indptr": "tensor_000104_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000104_chunk_000008_indices", "indptr": "tensor_000104_chunk_000008_indptr", "nnz": 205, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000104_chunk_000009_indices", "indptr": "tensor_000104_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000104_chunk_000010_indices", "indptr": "tensor_000104_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000104_chunk_000011_indices", "indptr": "tensor_000104_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 845, "shape": [ 6144, 2048 ] }, "model.layers.3.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000116_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.3.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000015_chunk_000000_indices", "indptr": "tensor_000015_chunk_000000_indptr", "nnz": 46, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000015_chunk_000001_indices", "indptr": "tensor_000015_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000015_chunk_000002_indices", "indptr": "tensor_000015_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000015_chunk_000003_indices", "indptr": "tensor_000015_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000015_chunk_000004_indices", "indptr": "tensor_000015_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000015_chunk_000005_indices", "indptr": "tensor_000015_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000015_chunk_000006_indices", "indptr": "tensor_000015_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 46, "shape": [ 2048, 3415 ] }, "model.layers.3.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000014_chunk_000000_indices", "indptr": "tensor_000014_chunk_000000_indptr", "nnz": 43, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000014_chunk_000001_indices", "indptr": "tensor_000014_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000014_chunk_000002_indices", "indptr": "tensor_000014_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000014_chunk_000003_indices", "indptr": "tensor_000014_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000014_chunk_000004_indices", "indptr": "tensor_000014_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000014_chunk_000005_indices", "indptr": "tensor_000014_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000014_chunk_000006_indices", "indptr": "tensor_000014_chunk_000006_indptr", "nnz": 25, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000014_chunk_000007_indices", "indptr": "tensor_000014_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000014_chunk_000008_indices", "indptr": "tensor_000014_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000014_chunk_000009_indices", "indptr": "tensor_000014_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000014_chunk_000010_indices", "indptr": "tensor_000014_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000014_chunk_000011_indices", "indptr": "tensor_000014_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000014_chunk_000012_indices", "indptr": "tensor_000014_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000014_chunk_000013_indices", "indptr": "tensor_000014_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 68, "shape": [ 6830, 2048 ] }, "model.layers.3.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000117_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.3.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000013_chunk_000000_indices", "indptr": "tensor_000013_chunk_000000_indptr", "nnz": 4, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000013_chunk_000001_indices", "indptr": "tensor_000013_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000013_chunk_000002_indices", "indptr": "tensor_000013_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000013_chunk_000003_indices", "indptr": "tensor_000013_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 4, "shape": [ 2048, 2048 ] }, "model.layers.3.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000012_chunk_000000_indices", "indptr": "tensor_000012_chunk_000000_indptr", "nnz": 192, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000012_chunk_000001_indices", "indptr": "tensor_000012_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000012_chunk_000002_indices", "indptr": "tensor_000012_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000012_chunk_000003_indices", "indptr": "tensor_000012_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000012_chunk_000004_indices", "indptr": "tensor_000012_chunk_000004_indptr", "nnz": 192, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000012_chunk_000005_indices", "indptr": "tensor_000012_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000012_chunk_000006_indices", "indptr": "tensor_000012_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000012_chunk_000007_indices", "indptr": "tensor_000012_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000012_chunk_000008_indices", "indptr": "tensor_000012_chunk_000008_indptr", "nnz": 4, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000012_chunk_000009_indices", "indptr": "tensor_000012_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000012_chunk_000010_indices", "indptr": "tensor_000012_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000012_chunk_000011_indices", "indptr": "tensor_000012_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 388, "shape": [ 6144, 2048 ] }, "model.layers.4.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000118_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.4.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000019_chunk_000000_indices", "indptr": "tensor_000019_chunk_000000_indptr", "nnz": 408, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000019_chunk_000001_indices", "indptr": "tensor_000019_chunk_000001_indptr", "nnz": 1, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000019_chunk_000002_indices", "indptr": "tensor_000019_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000019_chunk_000003_indices", "indptr": "tensor_000019_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000019_chunk_000004_indices", "indptr": "tensor_000019_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000019_chunk_000005_indices", "indptr": "tensor_000019_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000019_chunk_000006_indices", "indptr": "tensor_000019_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 409, "shape": [ 2048, 3415 ] }, "model.layers.4.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000018_chunk_000000_indices", "indptr": "tensor_000018_chunk_000000_indptr", "nnz": 400, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000018_chunk_000001_indices", "indptr": "tensor_000018_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000018_chunk_000002_indices", "indptr": "tensor_000018_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000018_chunk_000003_indices", "indptr": "tensor_000018_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000018_chunk_000004_indices", "indptr": "tensor_000018_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000018_chunk_000005_indices", "indptr": "tensor_000018_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000018_chunk_000006_indices", "indptr": "tensor_000018_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000018_chunk_000007_indices", "indptr": "tensor_000018_chunk_000007_indptr", "nnz": 141, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000018_chunk_000008_indices", "indptr": "tensor_000018_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000018_chunk_000009_indices", "indptr": "tensor_000018_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000018_chunk_000010_indices", "indptr": "tensor_000018_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000018_chunk_000011_indices", "indptr": "tensor_000018_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000018_chunk_000012_indices", "indptr": "tensor_000018_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000018_chunk_000013_indices", "indptr": "tensor_000018_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 710, "shape": [ 6830, 2048 ] }, "model.layers.4.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000119_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.4.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000017_chunk_000000_indices", "indptr": "tensor_000017_chunk_000000_indptr", "nnz": 28, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000017_chunk_000001_indices", "indptr": "tensor_000017_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000017_chunk_000002_indices", "indptr": "tensor_000017_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000017_chunk_000003_indices", "indptr": "tensor_000017_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 28, "shape": [ 2048, 2048 ] }, "model.layers.4.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000016_chunk_000000_indices", "indptr": "tensor_000016_chunk_000000_indptr", "nnz": 194, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000016_chunk_000001_indices", "indptr": "tensor_000016_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000016_chunk_000002_indices", "indptr": "tensor_000016_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000016_chunk_000003_indices", "indptr": "tensor_000016_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000016_chunk_000004_indices", "indptr": "tensor_000016_chunk_000004_indptr", "nnz": 194, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000016_chunk_000005_indices", "indptr": "tensor_000016_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000016_chunk_000006_indices", "indptr": "tensor_000016_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000016_chunk_000007_indices", "indptr": "tensor_000016_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000016_chunk_000008_indices", "indptr": "tensor_000016_chunk_000008_indptr", "nnz": 28, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000016_chunk_000009_indices", "indptr": "tensor_000016_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000016_chunk_000010_indices", "indptr": "tensor_000016_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000016_chunk_000011_indices", "indptr": "tensor_000016_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 416, "shape": [ 6144, 2048 ] }, "model.layers.5.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000120_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.5.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000023_chunk_000000_indices", "indptr": "tensor_000023_chunk_000000_indptr", "nnz": 304, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000023_chunk_000001_indices", "indptr": "tensor_000023_chunk_000001_indptr", "nnz": 71, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000023_chunk_000002_indices", "indptr": "tensor_000023_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000023_chunk_000003_indices", "indptr": "tensor_000023_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000023_chunk_000004_indices", "indptr": "tensor_000023_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000023_chunk_000005_indices", "indptr": "tensor_000023_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000023_chunk_000006_indices", "indptr": "tensor_000023_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 375, "shape": [ 2048, 3415 ] }, "model.layers.5.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000022_chunk_000000_indices", "indptr": "tensor_000022_chunk_000000_indptr", "nnz": 422, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000022_chunk_000001_indices", "indptr": "tensor_000022_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000022_chunk_000002_indices", "indptr": "tensor_000022_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000022_chunk_000003_indices", "indptr": "tensor_000022_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000022_chunk_000004_indices", "indptr": "tensor_000022_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000022_chunk_000005_indices", "indptr": "tensor_000022_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000022_chunk_000006_indices", "indptr": "tensor_000022_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000022_chunk_000007_indices", "indptr": "tensor_000022_chunk_000007_indptr", "nnz": 197, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000022_chunk_000008_indices", "indptr": "tensor_000022_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000022_chunk_000009_indices", "indptr": "tensor_000022_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000022_chunk_000010_indices", "indptr": "tensor_000022_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000022_chunk_000011_indices", "indptr": "tensor_000022_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000022_chunk_000012_indices", "indptr": "tensor_000022_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000022_chunk_000013_indices", "indptr": "tensor_000022_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 788, "shape": [ 6830, 2048 ] }, "model.layers.5.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000121_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.5.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000021_chunk_000000_indices", "indptr": "tensor_000021_chunk_000000_indptr", "nnz": 185, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000021_chunk_000001_indices", "indptr": "tensor_000021_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000021_chunk_000002_indices", "indptr": "tensor_000021_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000021_chunk_000003_indices", "indptr": "tensor_000021_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 185, "shape": [ 2048, 2048 ] }, "model.layers.5.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000020_chunk_000000_indices", "indptr": "tensor_000020_chunk_000000_indptr", "nnz": 321, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000020_chunk_000001_indices", "indptr": "tensor_000020_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000020_chunk_000002_indices", "indptr": "tensor_000020_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000020_chunk_000003_indices", "indptr": "tensor_000020_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000020_chunk_000004_indices", "indptr": "tensor_000020_chunk_000004_indptr", "nnz": 321, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000020_chunk_000005_indices", "indptr": "tensor_000020_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000020_chunk_000006_indices", "indptr": "tensor_000020_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000020_chunk_000007_indices", "indptr": "tensor_000020_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000020_chunk_000008_indices", "indptr": "tensor_000020_chunk_000008_indptr", "nnz": 185, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000020_chunk_000009_indices", "indptr": "tensor_000020_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000020_chunk_000010_indices", "indptr": "tensor_000020_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000020_chunk_000011_indices", "indptr": "tensor_000020_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 827, "shape": [ 6144, 2048 ] }, "model.layers.6.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000122_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.6.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000027_chunk_000000_indices", "indptr": "tensor_000027_chunk_000000_indptr", "nnz": 83, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000027_chunk_000001_indices", "indptr": "tensor_000027_chunk_000001_indptr", "nnz": 2, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000027_chunk_000002_indices", "indptr": "tensor_000027_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000027_chunk_000003_indices", "indptr": "tensor_000027_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000027_chunk_000004_indices", "indptr": "tensor_000027_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000027_chunk_000005_indices", "indptr": "tensor_000027_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000027_chunk_000006_indices", "indptr": "tensor_000027_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 85, "shape": [ 2048, 3415 ] }, "model.layers.6.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000026_chunk_000000_indices", "indptr": "tensor_000026_chunk_000000_indptr", "nnz": 86, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000026_chunk_000001_indices", "indptr": "tensor_000026_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000026_chunk_000002_indices", "indptr": "tensor_000026_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000026_chunk_000003_indices", "indptr": "tensor_000026_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000026_chunk_000004_indices", "indptr": "tensor_000026_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000026_chunk_000005_indices", "indptr": "tensor_000026_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000026_chunk_000006_indices", "indptr": "tensor_000026_chunk_000006_indptr", "nnz": 84, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000026_chunk_000007_indices", "indptr": "tensor_000026_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000026_chunk_000008_indices", "indptr": "tensor_000026_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000026_chunk_000009_indices", "indptr": "tensor_000026_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000026_chunk_000010_indices", "indptr": "tensor_000026_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000026_chunk_000011_indices", "indptr": "tensor_000026_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000026_chunk_000012_indices", "indptr": "tensor_000026_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000026_chunk_000013_indices", "indptr": "tensor_000026_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 170, "shape": [ 6830, 2048 ] }, "model.layers.6.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000123_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.6.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000025_chunk_000000_indices", "indptr": "tensor_000025_chunk_000000_indptr", "nnz": 2, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000025_chunk_000001_indices", "indptr": "tensor_000025_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000025_chunk_000002_indices", "indptr": "tensor_000025_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000025_chunk_000003_indices", "indptr": "tensor_000025_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2, "shape": [ 2048, 2048 ] }, "model.layers.6.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000024_chunk_000000_indices", "indptr": "tensor_000024_chunk_000000_indptr", "nnz": 128, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000024_chunk_000001_indices", "indptr": "tensor_000024_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000024_chunk_000002_indices", "indptr": "tensor_000024_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000024_chunk_000003_indices", "indptr": "tensor_000024_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000024_chunk_000004_indices", "indptr": "tensor_000024_chunk_000004_indptr", "nnz": 128, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000024_chunk_000005_indices", "indptr": "tensor_000024_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000024_chunk_000006_indices", "indptr": "tensor_000024_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000024_chunk_000007_indices", "indptr": "tensor_000024_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000024_chunk_000008_indices", "indptr": "tensor_000024_chunk_000008_indptr", "nnz": 2, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000024_chunk_000009_indices", "indptr": "tensor_000024_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000024_chunk_000010_indices", "indptr": "tensor_000024_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000024_chunk_000011_indices", "indptr": "tensor_000024_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 258, "shape": [ 6144, 2048 ] }, "model.layers.7.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000124_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.7.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000031_chunk_000000_indices", "indptr": "tensor_000031_chunk_000000_indptr", "nnz": 127, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000031_chunk_000001_indices", "indptr": "tensor_000031_chunk_000001_indptr", "nnz": 10, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000031_chunk_000002_indices", "indptr": "tensor_000031_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000031_chunk_000003_indices", "indptr": "tensor_000031_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000031_chunk_000004_indices", "indptr": "tensor_000031_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000031_chunk_000005_indices", "indptr": "tensor_000031_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000031_chunk_000006_indices", "indptr": "tensor_000031_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 137, "shape": [ 2048, 3415 ] }, "model.layers.7.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000030_chunk_000000_indices", "indptr": "tensor_000030_chunk_000000_indptr", "nnz": 107, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000030_chunk_000001_indices", "indptr": "tensor_000030_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000030_chunk_000002_indices", "indptr": "tensor_000030_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000030_chunk_000003_indices", "indptr": "tensor_000030_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000030_chunk_000004_indices", "indptr": "tensor_000030_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000030_chunk_000005_indices", "indptr": "tensor_000030_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000030_chunk_000006_indices", "indptr": "tensor_000030_chunk_000006_indptr", "nnz": 56, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000030_chunk_000007_indices", "indptr": "tensor_000030_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000030_chunk_000008_indices", "indptr": "tensor_000030_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000030_chunk_000009_indices", "indptr": "tensor_000030_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000030_chunk_000010_indices", "indptr": "tensor_000030_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000030_chunk_000011_indices", "indptr": "tensor_000030_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000030_chunk_000012_indices", "indptr": "tensor_000030_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000030_chunk_000013_indices", "indptr": "tensor_000030_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 163, "shape": [ 6830, 2048 ] }, "model.layers.7.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000125_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.7.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000029_chunk_000000_indices", "indptr": "tensor_000029_chunk_000000_indptr", "nnz": 26, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000029_chunk_000001_indices", "indptr": "tensor_000029_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000029_chunk_000002_indices", "indptr": "tensor_000029_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000029_chunk_000003_indices", "indptr": "tensor_000029_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 26, "shape": [ 2048, 2048 ] }, "model.layers.7.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000028_chunk_000000_indices", "indptr": "tensor_000028_chunk_000000_indptr", "nnz": 203, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000028_chunk_000001_indices", "indptr": "tensor_000028_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000028_chunk_000002_indices", "indptr": "tensor_000028_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000028_chunk_000003_indices", "indptr": "tensor_000028_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000028_chunk_000004_indices", "indptr": "tensor_000028_chunk_000004_indptr", "nnz": 203, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000028_chunk_000005_indices", "indptr": "tensor_000028_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000028_chunk_000006_indices", "indptr": "tensor_000028_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000028_chunk_000007_indices", "indptr": "tensor_000028_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000028_chunk_000008_indices", "indptr": "tensor_000028_chunk_000008_indptr", "nnz": 26, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000028_chunk_000009_indices", "indptr": "tensor_000028_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000028_chunk_000010_indices", "indptr": "tensor_000028_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000028_chunk_000011_indices", "indptr": "tensor_000028_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 432, "shape": [ 6144, 2048 ] }, "model.layers.8.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000126_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.8.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000035_chunk_000000_indices", "indptr": "tensor_000035_chunk_000000_indptr", "nnz": 24, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000035_chunk_000001_indices", "indptr": "tensor_000035_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000035_chunk_000002_indices", "indptr": "tensor_000035_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000035_chunk_000003_indices", "indptr": "tensor_000035_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000035_chunk_000004_indices", "indptr": "tensor_000035_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000035_chunk_000005_indices", "indptr": "tensor_000035_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000035_chunk_000006_indices", "indptr": "tensor_000035_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 24, "shape": [ 2048, 3415 ] }, "model.layers.8.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000034_chunk_000000_indices", "indptr": "tensor_000034_chunk_000000_indptr", "nnz": 22, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000034_chunk_000001_indices", "indptr": "tensor_000034_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000034_chunk_000002_indices", "indptr": "tensor_000034_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000034_chunk_000003_indices", "indptr": "tensor_000034_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000034_chunk_000004_indices", "indptr": "tensor_000034_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000034_chunk_000005_indices", "indptr": "tensor_000034_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000034_chunk_000006_indices", "indptr": "tensor_000034_chunk_000006_indptr", "nnz": 25, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000034_chunk_000007_indices", "indptr": "tensor_000034_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000034_chunk_000008_indices", "indptr": "tensor_000034_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000034_chunk_000009_indices", "indptr": "tensor_000034_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000034_chunk_000010_indices", "indptr": "tensor_000034_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000034_chunk_000011_indices", "indptr": "tensor_000034_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000034_chunk_000012_indices", "indptr": "tensor_000034_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000034_chunk_000013_indices", "indptr": "tensor_000034_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 47, "shape": [ 6830, 2048 ] }, "model.layers.8.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000127_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.8.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000033_chunk_000000_indices", "indptr": "tensor_000033_chunk_000000_indptr", "nnz": 88, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000033_chunk_000001_indices", "indptr": "tensor_000033_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000033_chunk_000002_indices", "indptr": "tensor_000033_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000033_chunk_000003_indices", "indptr": "tensor_000033_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 88, "shape": [ 2048, 2048 ] }, "model.layers.8.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000032_chunk_000000_indices", "indptr": "tensor_000032_chunk_000000_indptr", "nnz": 140, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000032_chunk_000001_indices", "indptr": "tensor_000032_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000032_chunk_000002_indices", "indptr": "tensor_000032_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000032_chunk_000003_indices", "indptr": "tensor_000032_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000032_chunk_000004_indices", "indptr": "tensor_000032_chunk_000004_indptr", "nnz": 140, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000032_chunk_000005_indices", "indptr": "tensor_000032_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000032_chunk_000006_indices", "indptr": "tensor_000032_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000032_chunk_000007_indices", "indptr": "tensor_000032_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000032_chunk_000008_indices", "indptr": "tensor_000032_chunk_000008_indptr", "nnz": 88, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000032_chunk_000009_indices", "indptr": "tensor_000032_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000032_chunk_000010_indices", "indptr": "tensor_000032_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000032_chunk_000011_indices", "indptr": "tensor_000032_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 368, "shape": [ 6144, 2048 ] }, "model.layers.9.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000128_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.9.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000039_chunk_000000_indices", "indptr": "tensor_000039_chunk_000000_indptr", "nnz": 6, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000039_chunk_000001_indices", "indptr": "tensor_000039_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000039_chunk_000002_indices", "indptr": "tensor_000039_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000039_chunk_000003_indices", "indptr": "tensor_000039_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000039_chunk_000004_indices", "indptr": "tensor_000039_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000039_chunk_000005_indices", "indptr": "tensor_000039_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000039_chunk_000006_indices", "indptr": "tensor_000039_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 6, "shape": [ 2048, 3415 ] }, "model.layers.9.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000038_chunk_000000_indices", "indptr": "tensor_000038_chunk_000000_indptr", "nnz": 6, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000038_chunk_000001_indices", "indptr": "tensor_000038_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000038_chunk_000002_indices", "indptr": "tensor_000038_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000038_chunk_000003_indices", "indptr": "tensor_000038_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000038_chunk_000004_indices", "indptr": "tensor_000038_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000038_chunk_000005_indices", "indptr": "tensor_000038_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000038_chunk_000006_indices", "indptr": "tensor_000038_chunk_000006_indptr", "nnz": 6, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000038_chunk_000007_indices", "indptr": "tensor_000038_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000038_chunk_000008_indices", "indptr": "tensor_000038_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000038_chunk_000009_indices", "indptr": "tensor_000038_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000038_chunk_000010_indices", "indptr": "tensor_000038_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000038_chunk_000011_indices", "indptr": "tensor_000038_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000038_chunk_000012_indices", "indptr": "tensor_000038_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000038_chunk_000013_indices", "indptr": "tensor_000038_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 12, "shape": [ 6830, 2048 ] }, "model.layers.9.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000129_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.9.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000037_chunk_000000_indices", "indptr": "tensor_000037_chunk_000000_indptr", "nnz": 4, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000037_chunk_000001_indices", "indptr": "tensor_000037_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000037_chunk_000002_indices", "indptr": "tensor_000037_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000037_chunk_000003_indices", "indptr": "tensor_000037_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 4, "shape": [ 2048, 2048 ] }, "model.layers.9.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000036_chunk_000000_indices", "indptr": "tensor_000036_chunk_000000_indptr", "nnz": 64, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000036_chunk_000001_indices", "indptr": "tensor_000036_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000036_chunk_000002_indices", "indptr": "tensor_000036_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000036_chunk_000003_indices", "indptr": "tensor_000036_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000036_chunk_000004_indices", "indptr": "tensor_000036_chunk_000004_indptr", "nnz": 64, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000036_chunk_000005_indices", "indptr": "tensor_000036_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000036_chunk_000006_indices", "indptr": "tensor_000036_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000036_chunk_000007_indices", "indptr": "tensor_000036_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000036_chunk_000008_indices", "indptr": "tensor_000036_chunk_000008_indptr", "nnz": 4, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000036_chunk_000009_indices", "indptr": "tensor_000036_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000036_chunk_000010_indices", "indptr": "tensor_000036_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000036_chunk_000011_indices", "indptr": "tensor_000036_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 132, "shape": [ 6144, 2048 ] }, "model.norm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000109_indices", "nnz": 2046, "shape": [ 2048 ] } }, "zero_test": "stored fp32 value != 0.0" }, "physical_layout": { "checkpoint_parameter_map": [ { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 64 ], "checkpoint_tensor": "model.layers.0.self_attn.qkv_proj.weight", "logical_matrix": "L0.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 64 ], "checkpoint_tensor": "model.layers.0.self_attn.qkv_proj.weight", "logical_matrix": "L0.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 64 ], "checkpoint_tensor": "model.layers.0.self_attn.qkv_proj.weight", "logical_matrix": "L0.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 64 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.0.self_attn.o_proj.weight", "logical_matrix": "L0.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 16 ], "checkpoint_tensor": "model.layers.0.mlp.gate_up_proj.weight", "logical_matrix": "L0.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 16 ], "checkpoint_tensor": "model.layers.0.mlp.gate_up_proj.weight", "logical_matrix": "L0.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 16 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.0.mlp.down_proj.weight", "logical_matrix": "L0.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.1.self_attn.qkv_proj.weight", "logical_matrix": "L1.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.1.self_attn.qkv_proj.weight", "logical_matrix": "L1.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.1.self_attn.qkv_proj.weight", "logical_matrix": "L1.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.1.self_attn.o_proj.weight", "logical_matrix": "L1.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 35 ], "checkpoint_tensor": "model.layers.1.mlp.gate_up_proj.weight", "logical_matrix": "L1.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 35 ], "checkpoint_tensor": "model.layers.1.mlp.gate_up_proj.weight", "logical_matrix": "L1.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 35 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.1.mlp.down_proj.weight", "logical_matrix": "L1.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 192 ], "checkpoint_tensor": "model.layers.2.self_attn.qkv_proj.weight", "logical_matrix": "L2.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 192 ], "checkpoint_tensor": "model.layers.2.self_attn.qkv_proj.weight", "logical_matrix": "L2.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 192 ], "checkpoint_tensor": "model.layers.2.self_attn.qkv_proj.weight", "logical_matrix": "L2.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 192 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.2.self_attn.o_proj.weight", "logical_matrix": "L2.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 73 ], "checkpoint_tensor": "model.layers.2.mlp.gate_up_proj.weight", "logical_matrix": "L2.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 73 ], "checkpoint_tensor": "model.layers.2.mlp.gate_up_proj.weight", "logical_matrix": "L2.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 73 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.2.mlp.down_proj.weight", "logical_matrix": "L2.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 192 ], "checkpoint_tensor": "model.layers.3.self_attn.qkv_proj.weight", "logical_matrix": "L3.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 192 ], "checkpoint_tensor": "model.layers.3.self_attn.qkv_proj.weight", "logical_matrix": "L3.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 192 ], "checkpoint_tensor": "model.layers.3.self_attn.qkv_proj.weight", "logical_matrix": "L3.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 192 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.3.self_attn.o_proj.weight", "logical_matrix": "L3.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 25 ], "checkpoint_tensor": "model.layers.3.mlp.gate_up_proj.weight", "logical_matrix": "L3.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 25 ], "checkpoint_tensor": "model.layers.3.mlp.gate_up_proj.weight", "logical_matrix": "L3.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 25 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.3.mlp.down_proj.weight", "logical_matrix": "L3.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 512 ], "checkpoint_tensor": "model.layers.4.self_attn.qkv_proj.weight", "logical_matrix": "L4.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 512 ], "checkpoint_tensor": "model.layers.4.self_attn.qkv_proj.weight", "logical_matrix": "L4.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 512 ], "checkpoint_tensor": "model.layers.4.self_attn.qkv_proj.weight", "logical_matrix": "L4.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 512 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.4.self_attn.o_proj.weight", "logical_matrix": "L4.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 310 ], "checkpoint_tensor": "model.layers.4.mlp.gate_up_proj.weight", "logical_matrix": "L4.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 310 ], "checkpoint_tensor": "model.layers.4.mlp.gate_up_proj.weight", "logical_matrix": "L4.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 310 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.4.mlp.down_proj.weight", "logical_matrix": "L4.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 384 ], "checkpoint_tensor": "model.layers.5.self_attn.qkv_proj.weight", "logical_matrix": "L5.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 384 ], "checkpoint_tensor": "model.layers.5.self_attn.qkv_proj.weight", "logical_matrix": "L5.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 384 ], "checkpoint_tensor": "model.layers.5.self_attn.qkv_proj.weight", "logical_matrix": "L5.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 384 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.5.self_attn.o_proj.weight", "logical_matrix": "L5.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 366 ], "checkpoint_tensor": "model.layers.5.mlp.gate_up_proj.weight", "logical_matrix": "L5.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 366 ], "checkpoint_tensor": "model.layers.5.mlp.gate_up_proj.weight", "logical_matrix": "L5.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 366 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.5.mlp.down_proj.weight", "logical_matrix": "L5.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 128 ], "checkpoint_tensor": "model.layers.6.self_attn.qkv_proj.weight", "logical_matrix": "L6.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 128 ], "checkpoint_tensor": "model.layers.6.self_attn.qkv_proj.weight", "logical_matrix": "L6.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 128 ], "checkpoint_tensor": "model.layers.6.self_attn.qkv_proj.weight", "logical_matrix": "L6.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 128 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.6.self_attn.o_proj.weight", "logical_matrix": "L6.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 84 ], "checkpoint_tensor": "model.layers.6.mlp.gate_up_proj.weight", "logical_matrix": "L6.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 84 ], "checkpoint_tensor": "model.layers.6.mlp.gate_up_proj.weight", "logical_matrix": "L6.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 84 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.6.mlp.down_proj.weight", "logical_matrix": "L6.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.7.self_attn.qkv_proj.weight", "logical_matrix": "L7.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.7.self_attn.qkv_proj.weight", "logical_matrix": "L7.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.7.self_attn.qkv_proj.weight", "logical_matrix": "L7.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.7.self_attn.o_proj.weight", "logical_matrix": "L7.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 56 ], "checkpoint_tensor": "model.layers.7.mlp.gate_up_proj.weight", "logical_matrix": "L7.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 56 ], "checkpoint_tensor": "model.layers.7.mlp.gate_up_proj.weight", "logical_matrix": "L7.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 56 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.7.mlp.down_proj.weight", "logical_matrix": "L7.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 320 ], "checkpoint_tensor": "model.layers.8.self_attn.qkv_proj.weight", "logical_matrix": "L8.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 320 ], "checkpoint_tensor": "model.layers.8.self_attn.qkv_proj.weight", "logical_matrix": "L8.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 320 ], "checkpoint_tensor": "model.layers.8.self_attn.qkv_proj.weight", "logical_matrix": "L8.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 320 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.8.self_attn.o_proj.weight", "logical_matrix": "L8.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 22 ], "checkpoint_tensor": "model.layers.8.mlp.gate_up_proj.weight", "logical_matrix": "L8.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 22 ], "checkpoint_tensor": "model.layers.8.mlp.gate_up_proj.weight", "logical_matrix": "L8.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 22 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.8.mlp.down_proj.weight", "logical_matrix": "L8.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 64 ], "checkpoint_tensor": "model.layers.9.self_attn.qkv_proj.weight", "logical_matrix": "L9.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 64 ], "checkpoint_tensor": "model.layers.9.self_attn.qkv_proj.weight", "logical_matrix": "L9.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 64 ], "checkpoint_tensor": "model.layers.9.self_attn.qkv_proj.weight", "logical_matrix": "L9.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 64 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.9.self_attn.o_proj.weight", "logical_matrix": "L9.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 7 ], "checkpoint_tensor": "model.layers.9.mlp.gate_up_proj.weight", "logical_matrix": "L9.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 7 ], "checkpoint_tensor": "model.layers.9.mlp.gate_up_proj.weight", "logical_matrix": "L9.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 7 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.9.mlp.down_proj.weight", "logical_matrix": "L9.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.10.self_attn.qkv_proj.weight", "logical_matrix": "L10.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.10.self_attn.qkv_proj.weight", "logical_matrix": "L10.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.10.self_attn.qkv_proj.weight", "logical_matrix": "L10.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.10.self_attn.o_proj.weight", "logical_matrix": "L10.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 9 ], "checkpoint_tensor": "model.layers.10.mlp.gate_up_proj.weight", "logical_matrix": "L10.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 9 ], "checkpoint_tensor": "model.layers.10.mlp.gate_up_proj.weight", "logical_matrix": "L10.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 9 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.10.mlp.down_proj.weight", "logical_matrix": "L10.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 512 ], "checkpoint_tensor": "model.layers.11.self_attn.qkv_proj.weight", "logical_matrix": "L11.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 512 ], "checkpoint_tensor": "model.layers.11.self_attn.qkv_proj.weight", "logical_matrix": "L11.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 512 ], "checkpoint_tensor": "model.layers.11.self_attn.qkv_proj.weight", "logical_matrix": "L11.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 512 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.11.self_attn.o_proj.weight", "logical_matrix": "L11.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1605 ], "checkpoint_tensor": "model.layers.11.mlp.gate_up_proj.weight", "logical_matrix": "L11.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 1605 ], "checkpoint_tensor": "model.layers.11.mlp.gate_up_proj.weight", "logical_matrix": "L11.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1605 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.11.mlp.down_proj.weight", "logical_matrix": "L11.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 128 ], "checkpoint_tensor": "model.layers.12.self_attn.qkv_proj.weight", "logical_matrix": "L12.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 128 ], "checkpoint_tensor": "model.layers.12.self_attn.qkv_proj.weight", "logical_matrix": "L12.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 128 ], "checkpoint_tensor": "model.layers.12.self_attn.qkv_proj.weight", "logical_matrix": "L12.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 128 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.12.self_attn.o_proj.weight", "logical_matrix": "L12.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 501 ], "checkpoint_tensor": "model.layers.12.mlp.gate_up_proj.weight", "logical_matrix": "L12.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 501 ], "checkpoint_tensor": "model.layers.12.mlp.gate_up_proj.weight", "logical_matrix": "L12.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 501 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.12.mlp.down_proj.weight", "logical_matrix": "L12.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 192 ], "checkpoint_tensor": "model.layers.13.self_attn.qkv_proj.weight", "logical_matrix": "L13.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 192 ], "checkpoint_tensor": "model.layers.13.self_attn.qkv_proj.weight", "logical_matrix": "L13.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 192 ], "checkpoint_tensor": "model.layers.13.self_attn.qkv_proj.weight", "logical_matrix": "L13.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 192 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.13.self_attn.o_proj.weight", "logical_matrix": "L13.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 643 ], "checkpoint_tensor": "model.layers.13.mlp.gate_up_proj.weight", "logical_matrix": "L13.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 643 ], "checkpoint_tensor": "model.layers.13.mlp.gate_up_proj.weight", "logical_matrix": "L13.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 643 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.13.mlp.down_proj.weight", "logical_matrix": "L13.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 128 ], "checkpoint_tensor": "model.layers.14.self_attn.qkv_proj.weight", "logical_matrix": "L14.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 128 ], "checkpoint_tensor": "model.layers.14.self_attn.qkv_proj.weight", "logical_matrix": "L14.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 128 ], "checkpoint_tensor": "model.layers.14.self_attn.qkv_proj.weight", "logical_matrix": "L14.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 128 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.14.self_attn.o_proj.weight", "logical_matrix": "L14.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 922 ], "checkpoint_tensor": "model.layers.14.mlp.gate_up_proj.weight", "logical_matrix": "L14.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 922 ], "checkpoint_tensor": "model.layers.14.mlp.gate_up_proj.weight", "logical_matrix": "L14.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 922 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.14.mlp.down_proj.weight", "logical_matrix": "L14.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 128 ], "checkpoint_tensor": "model.layers.15.self_attn.qkv_proj.weight", "logical_matrix": "L15.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 128 ], "checkpoint_tensor": "model.layers.15.self_attn.qkv_proj.weight", "logical_matrix": "L15.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 128 ], "checkpoint_tensor": "model.layers.15.self_attn.qkv_proj.weight", "logical_matrix": "L15.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 128 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.15.self_attn.o_proj.weight", "logical_matrix": "L15.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 797 ], "checkpoint_tensor": "model.layers.15.mlp.gate_up_proj.weight", "logical_matrix": "L15.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 797 ], "checkpoint_tensor": "model.layers.15.mlp.gate_up_proj.weight", "logical_matrix": "L15.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 797 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.15.mlp.down_proj.weight", "logical_matrix": "L15.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 128 ], "checkpoint_tensor": "model.layers.16.self_attn.qkv_proj.weight", "logical_matrix": "L16.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 128 ], "checkpoint_tensor": "model.layers.16.self_attn.qkv_proj.weight", "logical_matrix": "L16.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 128 ], "checkpoint_tensor": "model.layers.16.self_attn.qkv_proj.weight", "logical_matrix": "L16.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 128 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.16.self_attn.o_proj.weight", "logical_matrix": "L16.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 789 ], "checkpoint_tensor": "model.layers.16.mlp.gate_up_proj.weight", "logical_matrix": "L16.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 789 ], "checkpoint_tensor": "model.layers.16.mlp.gate_up_proj.weight", "logical_matrix": "L16.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 789 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.16.mlp.down_proj.weight", "logical_matrix": "L16.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 192 ], "checkpoint_tensor": "model.layers.17.self_attn.qkv_proj.weight", "logical_matrix": "L17.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 192 ], "checkpoint_tensor": "model.layers.17.self_attn.qkv_proj.weight", "logical_matrix": "L17.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 192 ], "checkpoint_tensor": "model.layers.17.self_attn.qkv_proj.weight", "logical_matrix": "L17.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 192 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.17.self_attn.o_proj.weight", "logical_matrix": "L17.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 676 ], "checkpoint_tensor": "model.layers.17.mlp.gate_up_proj.weight", "logical_matrix": "L17.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 676 ], "checkpoint_tensor": "model.layers.17.mlp.gate_up_proj.weight", "logical_matrix": "L17.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 676 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.17.mlp.down_proj.weight", "logical_matrix": "L17.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.18.self_attn.qkv_proj.weight", "logical_matrix": "L18.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.18.self_attn.qkv_proj.weight", "logical_matrix": "L18.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.18.self_attn.qkv_proj.weight", "logical_matrix": "L18.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.18.self_attn.o_proj.weight", "logical_matrix": "L18.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 180 ], "checkpoint_tensor": "model.layers.18.mlp.gate_up_proj.weight", "logical_matrix": "L18.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 180 ], "checkpoint_tensor": "model.layers.18.mlp.gate_up_proj.weight", "logical_matrix": "L18.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 180 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.18.mlp.down_proj.weight", "logical_matrix": "L18.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.19.self_attn.qkv_proj.weight", "logical_matrix": "L19.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.19.self_attn.qkv_proj.weight", "logical_matrix": "L19.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.19.self_attn.qkv_proj.weight", "logical_matrix": "L19.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.19.self_attn.o_proj.weight", "logical_matrix": "L19.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 80 ], "checkpoint_tensor": "model.layers.19.mlp.gate_up_proj.weight", "logical_matrix": "L19.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 80 ], "checkpoint_tensor": "model.layers.19.mlp.gate_up_proj.weight", "logical_matrix": "L19.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 80 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.19.mlp.down_proj.weight", "logical_matrix": "L19.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 128 ], "checkpoint_tensor": "model.layers.20.self_attn.qkv_proj.weight", "logical_matrix": "L20.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 128 ], "checkpoint_tensor": "model.layers.20.self_attn.qkv_proj.weight", "logical_matrix": "L20.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 128 ], "checkpoint_tensor": "model.layers.20.self_attn.qkv_proj.weight", "logical_matrix": "L20.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 128 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.20.self_attn.o_proj.weight", "logical_matrix": "L20.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 23 ], "checkpoint_tensor": "model.layers.20.mlp.gate_up_proj.weight", "logical_matrix": "L20.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 23 ], "checkpoint_tensor": "model.layers.20.mlp.gate_up_proj.weight", "logical_matrix": "L20.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 23 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.20.mlp.down_proj.weight", "logical_matrix": "L20.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 64 ], "checkpoint_tensor": "model.layers.21.self_attn.qkv_proj.weight", "logical_matrix": "L21.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 64 ], "checkpoint_tensor": "model.layers.21.self_attn.qkv_proj.weight", "logical_matrix": "L21.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 64 ], "checkpoint_tensor": "model.layers.21.self_attn.qkv_proj.weight", "logical_matrix": "L21.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 64 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.21.self_attn.o_proj.weight", "logical_matrix": "L21.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 13 ], "checkpoint_tensor": "model.layers.21.mlp.gate_up_proj.weight", "logical_matrix": "L21.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 13 ], "checkpoint_tensor": "model.layers.21.mlp.gate_up_proj.weight", "logical_matrix": "L21.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 13 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.21.mlp.down_proj.weight", "logical_matrix": "L21.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 128 ], "checkpoint_tensor": "model.layers.22.self_attn.qkv_proj.weight", "logical_matrix": "L22.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 128 ], "checkpoint_tensor": "model.layers.22.self_attn.qkv_proj.weight", "logical_matrix": "L22.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 128 ], "checkpoint_tensor": "model.layers.22.self_attn.qkv_proj.weight", "logical_matrix": "L22.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 128 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.22.self_attn.o_proj.weight", "logical_matrix": "L22.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 5 ], "checkpoint_tensor": "model.layers.22.mlp.gate_up_proj.weight", "logical_matrix": "L22.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 5 ], "checkpoint_tensor": "model.layers.22.mlp.gate_up_proj.weight", "logical_matrix": "L22.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 5 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.22.mlp.down_proj.weight", "logical_matrix": "L22.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 64 ], "checkpoint_tensor": "model.layers.23.self_attn.qkv_proj.weight", "logical_matrix": "L23.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 64 ], "checkpoint_tensor": "model.layers.23.self_attn.qkv_proj.weight", "logical_matrix": "L23.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 64 ], "checkpoint_tensor": "model.layers.23.self_attn.qkv_proj.weight", "logical_matrix": "L23.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 64 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.23.self_attn.o_proj.weight", "logical_matrix": "L23.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 47 ], "checkpoint_tensor": "model.layers.23.mlp.gate_up_proj.weight", "logical_matrix": "L23.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 47 ], "checkpoint_tensor": "model.layers.23.mlp.gate_up_proj.weight", "logical_matrix": "L23.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 47 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.23.mlp.down_proj.weight", "logical_matrix": "L23.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 64 ], "checkpoint_tensor": "model.layers.24.self_attn.qkv_proj.weight", "logical_matrix": "L24.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 64 ], "checkpoint_tensor": "model.layers.24.self_attn.qkv_proj.weight", "logical_matrix": "L24.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 64 ], "checkpoint_tensor": "model.layers.24.self_attn.qkv_proj.weight", "logical_matrix": "L24.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 64 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.24.self_attn.o_proj.weight", "logical_matrix": "L24.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1963 ], "checkpoint_tensor": "model.layers.24.mlp.gate_up_proj.weight", "logical_matrix": "L24.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 1963 ], "checkpoint_tensor": "model.layers.24.mlp.gate_up_proj.weight", "logical_matrix": "L24.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1963 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.24.mlp.down_proj.weight", "logical_matrix": "L24.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.25.self_attn.qkv_proj.weight", "logical_matrix": "L25.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 2048 ], "checkpoint_tensor": "model.layers.25.self_attn.qkv_proj.weight", "logical_matrix": "L25.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 2048 ], "checkpoint_tensor": "model.layers.25.self_attn.qkv_proj.weight", "logical_matrix": "L25.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.25.self_attn.o_proj.weight", "logical_matrix": "L25.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 3415 ], "checkpoint_tensor": "model.layers.25.mlp.gate_up_proj.weight", "logical_matrix": "L25.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 3415 ], "checkpoint_tensor": "model.layers.25.mlp.gate_up_proj.weight", "logical_matrix": "L25.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 3415 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.25.mlp.down_proj.weight", "logical_matrix": "L25.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 320 ], "checkpoint_tensor": "model.layers.26.self_attn.qkv_proj.weight", "logical_matrix": "L26.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 320 ], "checkpoint_tensor": "model.layers.26.self_attn.qkv_proj.weight", "logical_matrix": "L26.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 320 ], "checkpoint_tensor": "model.layers.26.self_attn.qkv_proj.weight", "logical_matrix": "L26.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 320 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.26.self_attn.o_proj.weight", "logical_matrix": "L26.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 479 ], "checkpoint_tensor": "model.layers.26.mlp.gate_up_proj.weight", "logical_matrix": "L26.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 479 ], "checkpoint_tensor": "model.layers.26.mlp.gate_up_proj.weight", "logical_matrix": "L26.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 479 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.26.mlp.down_proj.weight", "logical_matrix": "L26.mlp.W_out", "transform": "transpose" }, { "checkpoint_tensor": "model.embed_tokens.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.0.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.0.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.1.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.1.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.10.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.10.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.11.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.11.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.12.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.12.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.13.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.13.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.14.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.14.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.15.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.15.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.16.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.16.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.17.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.17.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.18.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.18.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.19.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.19.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.2.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.2.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.20.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.20.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.21.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.21.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.22.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.22.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.23.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.23.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.24.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.24.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.25.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.25.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.26.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.26.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.3.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.3.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.4.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.4.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.5.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.5.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.6.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.6.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.7.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.7.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.8.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.8.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.9.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.9.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.norm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" } ], "known_null_components": { "meaning": "These components are statically absent or padding. This is not an input-specific causal-relevance claim.", "never_referenced_residual_columns": [ [ 2002, 44 ] ], "padded_attention_heads": [ { "head_range": [ 1, 31 ], "layer": 0 }, { "head_range": [ 4, 28 ], "layer": 1 }, { "head_range": [ 3, 29 ], "layer": 2 }, { "head_range": [ 3, 29 ], "layer": 3 }, { "head_range": [ 8, 24 ], "layer": 4 }, { "head_range": [ 6, 26 ], "layer": 5 }, { "head_range": [ 2, 30 ], "layer": 6 }, { "head_range": [ 4, 28 ], "layer": 7 }, { "head_range": [ 5, 27 ], "layer": 8 }, { "head_range": [ 1, 31 ], "layer": 9 }, { "head_range": [ 4, 28 ], "layer": 10 }, { "head_range": [ 8, 24 ], "layer": 11 }, { "head_range": [ 2, 30 ], "layer": 12 }, { "head_range": [ 3, 29 ], "layer": 13 }, { "head_range": [ 2, 30 ], "layer": 14 }, { "head_range": [ 2, 30 ], "layer": 15 }, { "head_range": [ 2, 30 ], "layer": 16 }, { "head_range": [ 3, 29 ], "layer": 17 }, { "head_range": [ 4, 28 ], "layer": 18 }, { "head_range": [ 4, 28 ], "layer": 19 }, { "head_range": [ 2, 30 ], "layer": 20 }, { "head_range": [ 1, 31 ], "layer": 21 }, { "head_range": [ 2, 30 ], "layer": 22 }, { "head_range": [ 1, 31 ], "layer": 23 }, { "head_range": [ 1, 31 ], "layer": 24 }, { "head_range": [ 5, 27 ], "layer": 26 } ], "padded_mlp_neurons": [ { "layer": 0, "neuron_range": [ 16, 3399 ] }, { "layer": 1, "neuron_range": [ 35, 3380 ] }, { "layer": 2, "neuron_range": [ 73, 3342 ] }, { "layer": 3, "neuron_range": [ 25, 3390 ] }, { "layer": 4, "neuron_range": [ 310, 3105 ] }, { "layer": 5, "neuron_range": [ 366, 3049 ] }, { "layer": 6, "neuron_range": [ 84, 3331 ] }, { "layer": 7, "neuron_range": [ 56, 3359 ] }, { "layer": 8, "neuron_range": [ 22, 3393 ] }, { "layer": 9, "neuron_range": [ 7, 3408 ] }, { "layer": 10, "neuron_range": [ 9, 3406 ] }, { "layer": 11, "neuron_range": [ 1605, 1810 ] }, { "layer": 12, "neuron_range": [ 501, 2914 ] }, { "layer": 13, "neuron_range": [ 643, 2772 ] }, { "layer": 14, "neuron_range": [ 922, 2493 ] }, { "layer": 15, "neuron_range": [ 797, 2618 ] }, { "layer": 16, "neuron_range": [ 789, 2626 ] }, { "layer": 17, "neuron_range": [ 676, 2739 ] }, { "layer": 18, "neuron_range": [ 180, 3235 ] }, { "layer": 19, "neuron_range": [ 80, 3335 ] }, { "layer": 20, "neuron_range": [ 23, 3392 ] }, { "layer": 21, "neuron_range": [ 13, 3402 ] }, { "layer": 22, "neuron_range": [ 5, 3410 ] }, { "layer": 23, "neuron_range": [ 47, 3368 ] }, { "layer": 24, "neuron_range": [ 1963, 1452 ] }, { "layer": 26, "neuron_range": [ 479, 2936 ] } ] }, "matrices": { "L0.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L0.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 64, 2048 ] }, "L0.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L0.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L0.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 16 ] }, "L0.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 16, 2048 ] }, "L0.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 16 ] }, "L1.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L1.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L1.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L1.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L1.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 35 ] }, "L1.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 35, 2048 ] }, "L1.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 35 ] }, "L10.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L10.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L10.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L10.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L10.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 9 ] }, "L10.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 9, 2048 ] }, "L10.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 9 ] }, "L11.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 512 ] }, "L11.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 512, 2048 ] }, "L11.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 512 ] }, "L11.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 512 ] }, "L11.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1605 ] }, "L11.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 1605, 2048 ] }, "L11.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1605 ] }, "L12.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L12.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 128, 2048 ] }, "L12.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L12.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L12.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 501 ] }, "L12.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 501, 2048 ] }, "L12.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 501 ] }, "L13.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L13.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 192, 2048 ] }, "L13.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L13.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L13.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 643 ] }, "L13.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 643, 2048 ] }, "L13.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 643 ] }, "L14.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L14.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 128, 2048 ] }, "L14.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L14.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L14.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 922 ] }, "L14.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 922, 2048 ] }, "L14.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 922 ] }, "L15.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L15.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 128, 2048 ] }, "L15.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L15.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L15.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 797 ] }, "L15.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 797, 2048 ] }, "L15.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 797 ] }, "L16.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L16.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 128, 2048 ] }, "L16.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L16.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L16.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 789 ] }, "L16.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 789, 2048 ] }, "L16.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 789 ] }, "L17.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L17.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 192, 2048 ] }, "L17.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L17.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L17.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 676 ] }, "L17.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 676, 2048 ] }, "L17.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 676 ] }, "L18.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L18.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L18.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L18.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L18.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 180 ] }, "L18.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 180, 2048 ] }, "L18.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 180 ] }, "L19.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L19.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L19.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L19.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L19.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 80 ] }, "L19.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 80, 2048 ] }, "L19.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 80 ] }, "L2.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L2.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 192, 2048 ] }, "L2.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L2.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L2.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 73 ] }, "L2.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 73, 2048 ] }, "L2.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 73 ] }, "L20.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L20.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 128, 2048 ] }, "L20.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L20.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L20.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 23 ] }, "L20.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 23, 2048 ] }, "L20.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 23 ] }, "L21.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L21.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 64, 2048 ] }, "L21.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L21.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L21.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 13 ] }, "L21.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 13, 2048 ] }, "L21.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 13 ] }, "L22.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L22.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 128, 2048 ] }, "L22.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L22.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L22.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 5 ] }, "L22.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 5, 2048 ] }, "L22.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 5 ] }, "L23.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L23.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 64, 2048 ] }, "L23.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L23.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L23.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 47 ] }, "L23.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 47, 2048 ] }, "L23.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 47 ] }, "L24.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L24.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 64, 2048 ] }, "L24.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L24.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L24.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1963 ] }, "L24.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 1963, 2048 ] }, "L24.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1963 ] }, "L25.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L25.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 2048, 2048 ] }, "L25.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L25.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L25.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 3415 ] }, "L25.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 3415, 2048 ] }, "L25.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 3415 ] }, "L26.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 320 ] }, "L26.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 320, 2048 ] }, "L26.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 320 ] }, "L26.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 320 ] }, "L26.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 479 ] }, "L26.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 479, 2048 ] }, "L26.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 479 ] }, "L3.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L3.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 192, 2048 ] }, "L3.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L3.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L3.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 25 ] }, "L3.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 25, 2048 ] }, "L3.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 25 ] }, "L4.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 512 ] }, "L4.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 512, 2048 ] }, "L4.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 512 ] }, "L4.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 512 ] }, "L4.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 310 ] }, "L4.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 310, 2048 ] }, "L4.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 310 ] }, "L5.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 384 ] }, "L5.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 384, 2048 ] }, "L5.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 384 ] }, "L5.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 384 ] }, "L5.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 366 ] }, "L5.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 366, 2048 ] }, "L5.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 366 ] }, "L6.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L6.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 128, 2048 ] }, "L6.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L6.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L6.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 84 ] }, "L6.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 84, 2048 ] }, "L6.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 84 ] }, "L7.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L7.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L7.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L7.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L7.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 56 ] }, "L7.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 56, 2048 ] }, "L7.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 56 ] }, "L8.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 320 ] }, "L8.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 320, 2048 ] }, "L8.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 320 ] }, "L8.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 320 ] }, "L8.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 22 ] }, "L8.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 22, 2048 ] }, "L8.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 22 ] }, "L9.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L9.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 64, 2048 ] }, "L9.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L9.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L9.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 7 ] }, "L9.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 7, 2048 ] }, "L9.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 7 ] } }, "matrix_coordinates": "logical x@W compiler coordinates", "placements": { "i:0": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L26.mlp.W_out" } ], "l:0": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 29, 2 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 57, 4 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 285, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 5 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 296, 5 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 393, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1631, 3 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1646, 3 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1650, 7 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1694, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 29, 2 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 57, 4 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 285, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 288, 5 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 296, 5 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 337, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 393, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1631, 3 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1646, 3 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1650, 7 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1694, 1 ], "axis1": [ 1, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1, 34 ], "axis1": [ 1, 17 ], "matrix": "L26.mlp.W_out" } ], "l:10": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L26.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L26.attn.W_K" }, { "axis0": [ 40, 3 ], "axis1": [ 0, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 53, 4 ], "axis1": [ 0, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 76, 2 ], "axis1": [ 0, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 197, 15 ], "axis1": [ 0, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 213, 3 ], "axis1": [ 0, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 334, 1 ], "axis1": [ 0, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 404, 1 ], "axis1": [ 0, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 1895, 35 ], "axis1": [ 0, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 30, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 141, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 145, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 1631, 3 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 1646, 3 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 1650, 7 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 1694, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L26.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L26.attn.W_K" }, { "axis0": [ 1930, 64 ], "axis1": [ 64, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 30, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 141, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 145, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 1631, 3 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 1646, 3 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 1650, 7 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 1694, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L26.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L26.attn.W_K" }, { "axis0": [ 1994, 8 ], "axis1": [ 128, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 30, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 141, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 145, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 1631, 3 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 1646, 3 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 1650, 7 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 1694, 1 ], "matrix": "L26.attn.W_O" } ], "l:100": [ { "axis0": [ 27, 2 ], "axis1": [ 64, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 22, 1 ], "axis1": [ 64, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 29, 1 ], "axis1": [ 64, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 22, 1 ], "axis1": [ 64, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 28, 1 ], "matrix": "L1.attn.W_O" } ], "l:102": [ { "axis0": [ 0, 1 ], "axis1": [ 27, 1 ], "matrix": "L0.mlp.W_out" } ], "l:103": [ { "axis0": [ 0, 1 ], "axis1": [ 28, 1 ], "matrix": "L0.mlp.W_out" } ], "l:105": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 22, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 5, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 5, 1 ], "axis1": [ 22, 1 ], "matrix": "L0.mlp.W_out" } ], "l:106": [ { "axis0": [ 0, 1 ], "axis1": [ 29, 1 ], "matrix": "L0.mlp.W_out" } ], "l:107": [ { "axis0": [ 0, 1 ], "axis1": [ 15, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 22, 1 ], "axis1": [ 15, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 22, 1 ], "axis1": [ 16, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 15, 1 ], "axis1": [ 117, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 16, 1 ], "axis1": [ 117, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 117, 1 ], "matrix": "L1.mlp.W_out" } ], "l:108": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 52, 1 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 72, 3 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 113, 4 ], "matrix": "L7.attn.W_O" } ], "l:110": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 34, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 154, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 34, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 38, 2 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 154, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1, 3 ], "axis1": [ 72, 3 ], "matrix": "L6.mlp.W_out" } ], "l:112": [ { "axis0": [ 0, 1 ], "axis1": [ 33, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 33, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 154, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 20, 1 ], "axis1": [ 33, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 23, 1 ], "axis1": [ 33, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 33, 2 ], "axis1": [ 154, 1 ], "matrix": "L5.mlp.W_out" } ], "l:114": [ { "axis0": [ 0, 1 ], "axis1": [ 6, 10 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 6, 10 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 23, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 6, 10 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 6, 10 ], "axis1": [ 23, 1 ], "matrix": "L0.mlp.W_out" } ], "l:115": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 20, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 37, 1 ], "axis1": [ 1, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 20, 1 ], "matrix": "L4.mlp.W_out" } ], "l:116": [ { "axis0": [ 35, 2 ], "axis1": [ 64, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 33, 1 ], "axis1": [ 64, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 37, 1 ], "axis1": [ 64, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 33, 1 ], "axis1": [ 64, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 37, 1 ], "matrix": "L4.attn.W_O" } ], "l:118": [ { "axis0": [ 0, 1 ], "axis1": [ 35, 1 ], "matrix": "L0.mlp.W_out" } ], "l:119": [ { "axis0": [ 0, 1 ], "axis1": [ 36, 1 ], "matrix": "L0.mlp.W_out" } ], "l:12": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 339, 1 ], "axis1": [ 1, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 426, 17 ], "axis1": [ 1, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 1, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 426, 17 ], "axis1": [ 1, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 1, 17 ], "axis1": [ 40, 2 ], "matrix": "L25.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 197, 14 ], "matrix": "L25.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 404, 1 ], "matrix": "L25.mlp.W_out" } ], "l:121": [ { "axis0": [ 0, 1 ], "axis1": [ 37, 1 ], "matrix": "L0.mlp.W_out" } ], "l:122": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 22, 1 ], "axis1": [ 64, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 111, 1 ], "axis1": [ 64, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 117, 1 ], "axis1": [ 64, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 34, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 38, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 34, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 38, 2 ], "matrix": "L5.mlp.W_out" } ], "l:123": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 12 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 12 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 12 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 22, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 111, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 117, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 19, 1 ], "axis1": [ 3, 12 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 108, 1 ], "axis1": [ 3, 12 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 3, 12 ], "axis1": [ 22, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 3, 12 ], "axis1": [ 111, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 3, 12 ], "axis1": [ 117, 1 ], "matrix": "L4.mlp.W_out" } ], "l:125": [ { "axis0": [ 0, 1 ], "axis1": [ 15, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 24, 1 ], "axis1": [ 15, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 24, 1 ], "axis1": [ 16, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 15, 1 ], "axis1": [ 108, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 16, 1 ], "axis1": [ 108, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 15, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 16, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 108, 1 ], "matrix": "L3.mlp.W_out" } ], "l:126": [ { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 108, 1 ], "axis1": [ 384, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 19, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 19, 1 ], "matrix": "L4.mlp.W_out" } ], "l:128": [ { "axis0": [ 0, 1 ], "axis1": [ 11, 4 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 11, 4 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 22, 1 ], "axis1": [ 11, 4 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 38, 4 ], "axis1": [ 11, 4 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 22, 1 ], "axis1": [ 11, 4 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 38, 4 ], "axis1": [ 11, 4 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 11, 4 ], "axis1": [ 113, 4 ], "matrix": "L1.mlp.W_out" } ], "l:130": [ { "axis0": [ 0, 1 ], "axis1": [ 38, 4 ], "matrix": "L0.mlp.W_out" } ], "l:131": [ { "axis0": [ 0, 1 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1, 17 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 53, 17 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 154, 1 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1, 17 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 53, 17 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 154, 1 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 4, 34 ], "axis1": [ 19, 2 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 4, 34 ], "axis1": [ 75, 13 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 4, 34 ], "axis1": [ 111, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 4, 34 ], "axis1": [ 329, 1 ], "matrix": "L6.mlp.W_out" } ], "l:133": [ { "axis0": [ 0, 1 ], "axis1": [ 53, 17 ], "matrix": "L0.mlp.W_out" } ], "l:134": [ { "axis0": [ 97, 2 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 99, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 23, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 54, 5 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 61, 4 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 68, 2 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 72, 3 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 117, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 330, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 26, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 66, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 113, 5 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 120, 7 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 330, 1 ], "matrix": "L11.attn.W_O" } ], "l:136": [ { "axis0": [ 0, 1 ], "axis1": [ 97, 1 ], "matrix": "L0.mlp.W_out" } ], "l:137": [ { "axis0": [ 0, 1 ], "axis1": [ 98, 1 ], "matrix": "L0.mlp.W_out" } ], "l:139": [ { "axis0": [ 0, 1 ], "axis1": [ 99, 1 ], "matrix": "L0.mlp.W_out" } ], "l:14": [ { "axis0": [ 0, 1 ], "axis1": [ 154, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 154, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 339, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 283, 2 ], "axis1": [ 154, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 154, 2 ], "axis1": [ 339, 1 ], "matrix": "L5.mlp.W_out" } ], "l:140": [ { "axis0": [ 23, 1 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 66, 1 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 117, 1 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 330, 1 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 121, 4 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 155, 17 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 23, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 54, 5 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 61, 4 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 68, 2 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 72, 3 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 117, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 330, 1 ], "matrix": "L10.attn.W_O" } ], "l:141": [ { "axis0": [ 0, 1 ], "axis1": [ 43, 9 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 43, 9 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 23, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 66, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 117, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 330, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 51, 1 ], "axis1": [ 43, 9 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 43, 9 ], "axis1": [ 23, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 43, 9 ], "axis1": [ 66, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 43, 9 ], "axis1": [ 117, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 43, 9 ], "axis1": [ 330, 1 ], "matrix": "L7.mlp.W_out" } ], "l:142": [ { "axis0": [ 49, 2 ], "axis1": [ 64, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 64, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 51, 1 ], "axis1": [ 64, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 71, 1 ], "axis1": [ 64, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 51, 1 ], "matrix": "L7.attn.W_O" } ], "l:144": [ { "axis0": [ 0, 1 ], "axis1": [ 49, 1 ], "matrix": "L0.mlp.W_out" } ], "l:145": [ { "axis0": [ 0, 1 ], "axis1": [ 50, 1 ], "matrix": "L0.mlp.W_out" } ], "l:147": [ { "axis0": [ 0, 1 ], "axis1": [ 51, 1 ], "matrix": "L0.mlp.W_out" } ], "l:148": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 153, 1 ], "axis1": [ 0, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 71, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 71, 1 ], "matrix": "L6.mlp.W_out" } ], "l:149": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 32 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 32 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 153, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 33, 1 ], "axis1": [ 1, 32 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1, 32 ], "axis1": [ 153, 1 ], "matrix": "L5.mlp.W_out" } ], "l:150": [ { "axis0": [ 34, 1 ], "axis1": [ 0, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 36, 1 ], "axis1": [ 0, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 296, 1 ], "axis1": [ 0, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 33, 1 ], "matrix": "L5.attn.W_O" } ], "l:151": [ { "axis0": [ 0, 1 ], "axis1": [ 34, 1 ], "matrix": "L0.mlp.W_out" } ], "l:152": [ { "axis0": [ 30, 2 ], "axis1": [ 0, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 32, 2 ], "axis1": [ 0, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 33, 1 ], "axis1": [ 0, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 36, 1 ], "matrix": "L4.attn.W_O" } ], "l:154": [ { "axis0": [ 0, 1 ], "axis1": [ 30, 1 ], "matrix": "L0.mlp.W_out" } ], "l:155": [ { "axis0": [ 0, 1 ], "axis1": [ 31, 1 ], "matrix": "L0.mlp.W_out" } ], "l:157": [ { "axis0": [ 0, 1 ], "axis1": [ 32, 1 ], "matrix": "L0.mlp.W_out" } ], "l:158": [ { "axis0": [ 0, 1 ], "axis1": [ 188, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 188, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 189, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 188, 1 ], "axis1": [ 296, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 189, 1 ], "axis1": [ 296, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 296, 1 ], "matrix": "L4.mlp.W_out" } ], "l:159": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 70, 1 ], "axis1": [ 0, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 258, 3 ], "axis1": [ 0, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 121, 4 ], "matrix": "L9.attn.W_O" } ], "l:16": [ { "axis0": [ 0, 1 ], "axis1": [ 162, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 162, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 283, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 162, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 162, 2 ], "axis1": [ 283, 1 ], "matrix": "L4.mlp.W_out" } ], "l:161": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 120, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 255, 3 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 120, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 255, 3 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1, 3 ], "axis1": [ 258, 3 ], "matrix": "L8.mlp.W_out" } ], "l:163": [ { "axis0": [ 0, 1 ], "axis1": [ 15, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 15, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 120, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 23, 1 ], "axis1": [ 15, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 37, 1 ], "axis1": [ 15, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 112, 1 ], "axis1": [ 15, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 15, 2 ], "axis1": [ 120, 1 ], "matrix": "L4.mlp.W_out" } ], "l:165": [ { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 112, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 27, 1 ], "axis1": [ 9, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 9, 2 ], "axis1": [ 112, 1 ], "matrix": "L1.mlp.W_out" } ], "l:166": [ { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 54, 3 ], "axis1": [ 192, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 255, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 255, 3 ], "matrix": "L8.mlp.W_out" } ], "l:167": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 54, 3 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 22, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 53, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1, 12 ], "axis1": [ 54, 3 ], "matrix": "L7.mlp.W_out" } ], "l:169": [ { "axis0": [ 0, 1 ], "axis1": [ 38, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 71, 1 ], "axis1": [ 38, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 71, 1 ], "axis1": [ 39, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 38, 1 ], "axis1": [ 22, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 39, 1 ], "axis1": [ 22, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 38, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 39, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 22, 1 ], "matrix": "L6.mlp.W_out" } ], "l:17": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 28, 1 ], "axis1": [ 64, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 25, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 25, 1 ], "matrix": "L3.mlp.W_out" } ], "l:170": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 22, 1 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 53, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 53, 1 ], "matrix": "L7.mlp.W_out" } ], "l:172": [ { "axis0": [ 0, 1 ], "axis1": [ 17, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 17, 4 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 17, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 42, 4 ], "axis1": [ 17, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 17, 4 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 42, 4 ], "axis1": [ 17, 4 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 17, 4 ], "axis1": [ 121, 4 ], "matrix": "L4.mlp.W_out" } ], "l:174": [ { "axis0": [ 0, 1 ], "axis1": [ 42, 4 ], "matrix": "L0.mlp.W_out" } ], "l:175": [ { "axis0": [ 0, 1 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1, 17 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 71, 17 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 120, 1 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1, 17 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 71, 17 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 120, 1 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 35, 34 ], "axis1": [ 155, 17 ], "matrix": "L5.mlp.W_out" } ], "l:177": [ { "axis0": [ 0, 1 ], "axis1": [ 71, 17 ], "matrix": "L0.mlp.W_out" } ], "l:178": [ { "axis0": [ 0, 1 ], "axis1": [ 201, 12 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 201, 12 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 351, 1 ], "axis1": [ 201, 12 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 201, 12 ], "axis1": [ 67, 1 ], "matrix": "L16.mlp.W_out" } ], "l:179": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 22, 1 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 351, 1 ], "matrix": "L16.attn.W_O" } ], "l:18": [ { "axis0": [ 0, 1 ], "axis1": [ 35, 32 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 32 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 28, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 35, 32 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 35, 32 ], "axis1": [ 28, 1 ], "matrix": "L2.mlp.W_out" } ], "l:180": [ { "axis0": [ 0, 1 ], "axis1": [ 455, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 53, 2 ], "axis1": [ 455, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 265, 2 ], "axis1": [ 455, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 271, 2 ], "axis1": [ 455, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 53, 2 ], "axis1": [ 456, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 265, 2 ], "axis1": [ 456, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 271, 2 ], "axis1": [ 456, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 455, 1 ], "axis1": [ 351, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 456, 1 ], "axis1": [ 351, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" } ], "l:182": [ { "axis0": [ 0, 1 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 30, 3 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 55, 4 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 61, 4 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 68, 8 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 153, 14 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 901, 100 ], "axis1": [ 265, 2 ], "matrix": "L11.mlp.W_out" } ], "l:184": [ { "axis0": [ 100, 2 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 102, 1 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 66, 1 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 120, 5 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 155, 11 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 30, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 153, 14 ], "matrix": "L11.attn.W_O" } ], "l:186": [ { "axis0": [ 0, 1 ], "axis1": [ 100, 1 ], "matrix": "L0.mlp.W_out" } ], "l:187": [ { "axis0": [ 0, 1 ], "axis1": [ 101, 1 ], "matrix": "L0.mlp.W_out" } ], "l:189": [ { "axis0": [ 0, 1 ], "axis1": [ 102, 1 ], "matrix": "L0.mlp.W_out" } ], "l:19": [ { "axis0": [ 119, 1 ], "axis1": [ 64, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 29, 1 ], "axis1": [ 64, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 118, 1 ], "axis1": [ 64, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 25, 1 ], "matrix": "L2.attn.W_O" } ], "l:190": [ { "axis0": [ 63, 2 ], "axis1": [ 64, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 68, 2 ], "axis1": [ 64, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 121, 4 ], "axis1": [ 64, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 155, 17 ], "axis1": [ 64, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 66, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 120, 5 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 155, 11 ], "matrix": "L10.attn.W_O" } ], "l:191": [ { "axis0": [ 0, 1 ], "axis1": [ 30, 13 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 30, 13 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 63, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 68, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 51, 1 ], "axis1": [ 30, 13 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 30, 13 ], "axis1": [ 63, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 30, 13 ], "axis1": [ 68, 2 ], "matrix": "L7.mlp.W_out" } ], "l:192": [ { "axis0": [ 0, 1 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 26, 3 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 76, 13 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 90, 3 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 329, 2 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1001, 100 ], "axis1": [ 53, 2 ], "matrix": "L11.mlp.W_out" } ], "l:194": [ { "axis0": [ 91, 2 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 93, 1 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 20, 1 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 22, 1 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 26, 3 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 30, 3 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 111, 1 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 113, 4 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 125, 2 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 153, 2 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 76, 13 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 90, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 329, 1 ], "matrix": "L11.attn.W_O" } ], "l:196": [ { "axis0": [ 0, 1 ], "axis1": [ 91, 1 ], "matrix": "L0.mlp.W_out" } ], "l:197": [ { "axis0": [ 0, 1 ], "axis1": [ 92, 1 ], "matrix": "L0.mlp.W_out" } ], "l:199": [ { "axis0": [ 0, 1 ], "axis1": [ 93, 1 ], "matrix": "L0.mlp.W_out" } ], "l:2": [ { "axis0": [ 107, 2 ], "axis1": [ 128, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 128, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 109, 1 ], "axis1": [ 128, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 21, 1 ], "axis1": [ 128, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 29, 1 ], "matrix": "L1.attn.W_O" } ], "l:20": [ { "axis0": [ 0, 1 ], "axis1": [ 119, 1 ], "matrix": "L1.mlp.W_out" } ], "l:200": [ { "axis0": [ 28, 1 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 30, 3 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 113, 4 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 19, 2 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 75, 13 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 111, 1 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 329, 1 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 20, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 22, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 26, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 30, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 111, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 113, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 125, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 153, 2 ], "matrix": "L8.attn.W_O" } ], "l:201": [ { "axis0": [ 0, 1 ], "axis1": [ 38, 13 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 38, 13 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 28, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 30, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 46, 1 ], "axis1": [ 38, 13 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 38, 13 ], "axis1": [ 28, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 38, 13 ], "axis1": [ 30, 3 ], "matrix": "L4.mlp.W_out" } ], "l:202": [ { "axis0": [ 0, 1 ], "axis1": [ 201, 12 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 201, 12 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 350, 1 ], "axis1": [ 201, 12 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 201, 12 ], "axis1": [ 22, 1 ], "matrix": "L15.mlp.W_out" } ], "l:203": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 323, 1 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 350, 1 ], "matrix": "L15.attn.W_O" } ], "l:204": [ { "axis0": [ 0, 1 ], "axis1": [ 453, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 53, 2 ], "axis1": [ 453, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 259, 8 ], "axis1": [ 453, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 53, 2 ], "axis1": [ 454, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 259, 8 ], "axis1": [ 454, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 453, 1 ], "axis1": [ 350, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 454, 1 ], "axis1": [ 350, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" } ], "l:206": [ { "axis0": [ 0, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 33, 3 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 55, 4 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 61, 4 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 68, 8 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 249, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 252, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 255, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 601, 100 ], "axis1": [ 259, 2 ], "matrix": "L11.mlp.W_out" } ], "l:208": [ { "axis0": [ 103, 2 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 105, 1 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 52, 2 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 166, 6 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 255, 3 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 261, 6 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 33, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 38, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 49, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 168, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 178, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 249, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 252, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 255, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 327, 1 ], "matrix": "L11.attn.W_O" } ], "l:21": [ { "axis0": [ 0, 1 ], "axis1": [ 17, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 21, 1 ], "axis1": [ 17, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 21, 1 ], "axis1": [ 18, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 17, 1 ], "axis1": [ 118, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 18, 1 ], "axis1": [ 118, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 118, 1 ], "matrix": "L1.mlp.W_out" } ], "l:210": [ { "axis0": [ 0, 1 ], "axis1": [ 103, 1 ], "matrix": "L0.mlp.W_out" } ], "l:211": [ { "axis0": [ 0, 1 ], "axis1": [ 104, 1 ], "matrix": "L0.mlp.W_out" } ], "l:213": [ { "axis0": [ 0, 1 ], "axis1": [ 105, 1 ], "matrix": "L0.mlp.W_out" } ], "l:214": [ { "axis0": [ 57, 2 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 61, 2 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 121, 4 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 155, 17 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 52, 2 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 166, 6 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 255, 3 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 261, 6 ], "matrix": "L10.attn.W_O" } ], "l:215": [ { "axis0": [ 0, 1 ], "axis1": [ 13, 17 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 13, 17 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 57, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 61, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 51, 1 ], "axis1": [ 13, 17 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 13, 17 ], "axis1": [ 57, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 13, 17 ], "axis1": [ 61, 2 ], "matrix": "L7.mlp.W_out" } ], "l:216": [ { "axis0": [ 0, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 30, 3 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 76, 13 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 90, 3 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 153, 14 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 701, 100 ], "axis1": [ 261, 2 ], "matrix": "L11.mlp.W_out" } ], "l:218": [ { "axis0": [ 0, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 19, 5 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 26, 3 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 93, 3 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 97, 4 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 801, 100 ], "axis1": [ 263, 2 ], "matrix": "L11.mlp.W_out" } ], "l:22": [ { "axis0": [ 0, 1 ], "axis1": [ 164, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 164, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 284, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 164, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 164, 2 ], "axis1": [ 284, 1 ], "matrix": "L4.mlp.W_out" } ], "l:220": [ { "axis0": [ 94, 2 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 96, 1 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 33, 3 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 38, 2 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 49, 2 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 178, 3 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 249, 6 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 327, 1 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 19, 5 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 93, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 97, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 102, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 111, 1 ], "matrix": "L11.attn.W_O" } ], "l:222": [ { "axis0": [ 0, 1 ], "axis1": [ 94, 1 ], "matrix": "L0.mlp.W_out" } ], "l:223": [ { "axis0": [ 0, 1 ], "axis1": [ 95, 1 ], "matrix": "L0.mlp.W_out" } ], "l:225": [ { "axis0": [ 0, 1 ], "axis1": [ 96, 1 ], "matrix": "L0.mlp.W_out" } ], "l:226": [ { "axis0": [ 26, 2 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 125, 2 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 113, 4 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 19, 2 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 75, 13 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 111, 1 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 329, 1 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 33, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 38, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 49, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 178, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 249, 6 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 327, 1 ], "matrix": "L8.attn.W_O" } ], "l:227": [ { "axis0": [ 0, 1 ], "axis1": [ 21, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 21, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 26, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 125, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 46, 1 ], "axis1": [ 21, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 21, 17 ], "axis1": [ 26, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 21, 17 ], "axis1": [ 125, 2 ], "matrix": "L4.mlp.W_out" } ], "l:228": [ { "axis0": [ 0, 1 ], "axis1": [ 69, 12 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 12 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 349, 1 ], "axis1": [ 69, 12 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 69, 12 ], "axis1": [ 323, 1 ], "matrix": "L14.mlp.W_out" } ], "l:229": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 50, 1 ], "axis1": [ 0, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 349, 1 ], "matrix": "L14.attn.W_O" } ], "l:23": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 43, 10 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 61, 5 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 67, 2 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 70, 6 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 78, 11 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 216, 1 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 218, 4 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 224, 8 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 233, 4 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 238, 4 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 243, 4 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 248, 3 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 252, 2 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 426, 17 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 254, 8 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 263, 1 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 265, 3 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 269, 16 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 286, 2 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 301, 6 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 308, 4 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 313, 4 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 318, 1 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 320, 3 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 324, 3 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 328, 2 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 332, 1 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 335, 2 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 338, 1 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 346, 7 ], "axis1": [ 64, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 426, 17 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 353, 5 ], "axis1": [ 128, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 359, 4 ], "axis1": [ 128, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 364, 3 ], "axis1": [ 128, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 369, 9 ], "axis1": [ 128, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 379, 13 ], "axis1": [ 128, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 394, 9 ], "axis1": [ 128, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 405, 7 ], "axis1": [ 128, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 414, 1 ], "axis1": [ 128, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 416, 10 ], "axis1": [ 128, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 443, 3 ], "axis1": [ 128, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 426, 17 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 446, 12 ], "axis1": [ 192, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 426, 17 ], "matrix": "L25.attn.W_O" } ], "l:230": [ { "axis0": [ 0, 1 ], "axis1": [ 451, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 253, 2 ], "axis1": [ 451, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 256, 2 ], "axis1": [ 451, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 259, 6 ], "axis1": [ 451, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 253, 2 ], "axis1": [ 452, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 256, 2 ], "axis1": [ 452, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 259, 6 ], "axis1": [ 452, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 451, 1 ], "axis1": [ 349, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 452, 1 ], "axis1": [ 349, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" } ], "l:232": [ { "axis0": [ 0, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 33, 3 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 76, 13 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 90, 3 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 249, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 252, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 255, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 301, 100 ], "axis1": [ 253, 2 ], "matrix": "L11.mlp.W_out" } ], "l:234": [ { "axis0": [ 0, 1 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 19, 5 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 30, 3 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 93, 3 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 97, 4 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 153, 14 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 401, 100 ], "axis1": [ 256, 2 ], "matrix": "L11.mlp.W_out" } ], "l:236": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 12 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 12 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 267, 1 ], "axis1": [ 3, 12 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 3, 12 ], "axis1": [ 50, 1 ], "matrix": "L13.mlp.W_out" } ], "l:237": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 106, 1 ], "axis1": [ 0, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 267, 1 ], "matrix": "L13.attn.W_O" } ], "l:238": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 51, 1 ], "axis1": [ 0, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 253, 2 ], "axis1": [ 0, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 256, 3 ], "axis1": [ 0, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 267, 1 ], "matrix": "L12.attn.W_O" } ], "l:24": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 18, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 21, 3 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 26, 3 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 30, 9 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 38, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 38, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 38, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 38, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 39, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 39, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 39, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 39, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 39, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 76, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 90, 6 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 153, 14 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 168, 11 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 178, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 178, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 178, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 178, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 178, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 179, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 179, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 179, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 179, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 179, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 180, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 180, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 180, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 180, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 180, 22 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 201, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 201, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 201, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 201, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 201, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 202, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 202, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 202, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 202, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 202, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 203, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 203, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 203, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 203, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 203, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 204, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 204, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 204, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 204, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 204, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 205, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 205, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 205, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 205, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 205, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 206, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 206, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 206, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 206, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 206, 6 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 213, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 213, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 213, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 213, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 213, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 213, 3 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 215, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 215, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 215, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 215, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 215, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 330, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 333, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 426, 17 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 18, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 21, 3 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 26, 3 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 30, 9 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 38, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 38, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 38, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 38, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 38, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 39, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 39, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 39, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 39, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 39, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 53, 4 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 66, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 76, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 90, 6 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 106, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 111, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 113, 5 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 120, 7 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 153, 14 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 168, 11 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 178, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 178, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 178, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 178, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 178, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 179, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 179, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 179, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 179, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 179, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 180, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 180, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 180, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 180, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 180, 22 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 201, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 201, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 201, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 201, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 201, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 202, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 202, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 202, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 202, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 202, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 203, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 203, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 203, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 203, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 203, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 204, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 204, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 204, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 204, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 204, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 205, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 205, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 205, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 205, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 205, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 206, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 206, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 206, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 206, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 206, 6 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 213, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 213, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 213, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 213, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 213, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 213, 3 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 215, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 215, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 215, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 215, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 215, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 330, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 333, 2 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 334, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 334, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 334, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 334, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 334, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 426, 17 ], "axis1": [ 1, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 1, 204 ], "axis1": [ 43, 10 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 61, 5 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 67, 2 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 70, 6 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 78, 11 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 216, 1 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 218, 4 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 224, 8 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 233, 4 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 238, 4 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 243, 4 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 248, 3 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 252, 10 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 263, 1 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 265, 3 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 269, 16 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 286, 2 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 301, 6 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 308, 4 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 313, 4 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 318, 1 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 320, 3 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 324, 3 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 328, 2 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 332, 1 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 335, 2 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 338, 1 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 346, 12 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 359, 4 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 364, 3 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 369, 9 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 379, 13 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 394, 9 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 405, 7 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 414, 1 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 416, 10 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 443, 15 ], "matrix": "L24.mlp.W_out" } ], "l:240": [ { "axis0": [ 0, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 19, 5 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 33, 3 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 93, 3 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 97, 4 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 249, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 252, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 255, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 501, 100 ], "axis1": [ 51, 1 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 501, 100 ], "axis1": [ 258, 1 ], "matrix": "L11.mlp.W_out" } ], "l:243": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 54, 1 ], "matrix": "L20.mlp.W_out" }, { "axis0": [ 50, 1 ], "axis1": [ 1, 2 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 204, 1 ], "axis1": [ 1, 2 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 54, 1 ], "matrix": "L20.mlp.W_out" } ], "l:245": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_K" }, { "axis0": [ 201, 1 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 50, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 50, 1 ], "matrix": "L20.mlp.W_out" } ], "l:246": [ { "axis0": [ 0, 1 ], "axis1": [ 4, 2 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 56, 1 ], "axis1": [ 4, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 56, 1 ], "axis1": [ 5, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 4, 1 ], "axis1": [ 204, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 5, 1 ], "axis1": [ 204, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 204, 1 ], "matrix": "L19.mlp.W_out" } ], "l:247": [ { "axis0": [ 0, 1 ], "axis1": [ 35, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 1 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 56, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 19, 1 ], "axis1": [ 35, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 35, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 30, 2 ], "axis1": [ 35, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 90, 6 ], "axis1": [ 35, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 35, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 153, 5 ], "axis1": [ 35, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 35, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 56, 1 ], "matrix": "L18.mlp.W_out" } ], "l:248": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_K" }, { "axis0": [ 42, 5 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 65, 1 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 207, 5 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 213, 4 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 218, 4 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 224, 8 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 233, 8 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 285, 5 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 324, 3 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 346, 3 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 351, 1 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 353, 4 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 19, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 28, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 30, 2 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 90, 6 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 106, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 153, 5 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 331, 1 ], "matrix": "L18.attn.W_O" } ], "l:250": [ { "axis0": [ 0, 1 ], "axis1": [ 103, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 103, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 47, 1 ], "axis1": [ 103, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 297, 17 ], "axis1": [ 103, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 47, 1 ], "axis1": [ 103, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 297, 17 ], "axis1": [ 103, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 103, 17 ], "axis1": [ 42, 3 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 103, 17 ], "axis1": [ 207, 5 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 103, 17 ], "axis1": [ 213, 4 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 103, 17 ], "axis1": [ 218, 4 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 103, 17 ], "axis1": [ 227, 1 ], "matrix": "L5.mlp.W_out" } ], "l:252": [ { "axis0": [ 0, 1 ], "axis1": [ 120, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 120, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 48, 1 ], "axis1": [ 120, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 314, 17 ], "axis1": [ 120, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 48, 1 ], "axis1": [ 120, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 314, 17 ], "axis1": [ 120, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 120, 17 ], "axis1": [ 45, 2 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 120, 17 ], "axis1": [ 224, 3 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 120, 17 ], "axis1": [ 228, 4 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 120, 17 ], "axis1": [ 233, 8 ], "matrix": "L5.mlp.W_out" } ], "l:254": [ { "axis0": [ 0, 1 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 53, 8 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 265, 2 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 269, 3 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 290, 3 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 297, 1 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 53, 8 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 265, 2 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 269, 3 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 290, 3 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 297, 1 ], "axis1": [ 263, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 263, 17 ], "axis1": [ 65, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 263, 17 ], "axis1": [ 285, 5 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 263, 17 ], "axis1": [ 324, 3 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 263, 17 ], "axis1": [ 346, 3 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 263, 17 ], "axis1": [ 351, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 263, 17 ], "axis1": [ 353, 4 ], "matrix": "L17.mlp.W_out" } ], "l:256": [ { "axis0": [ 0, 1 ], "axis1": [ 213, 134 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 213, 134 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 53, 8 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 265, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 269, 3 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 290, 3 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 297, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 351, 1 ], "axis1": [ 213, 134 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 213, 134 ], "axis1": [ 53, 8 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 213, 134 ], "axis1": [ 265, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 213, 134 ], "axis1": [ 269, 3 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 213, 134 ], "axis1": [ 290, 3 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 213, 134 ], "axis1": [ 297, 1 ], "matrix": "L16.mlp.W_out" } ], "l:257": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 55, 1 ], "matrix": "L20.mlp.W_out" }, { "axis0": [ 50, 1 ], "axis1": [ 3, 2 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 204, 1 ], "axis1": [ 3, 2 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 286, 1 ], "axis1": [ 3, 2 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 3, 2 ], "axis1": [ 55, 1 ], "matrix": "L20.mlp.W_out" } ], "l:259": [ { "axis0": [ 0, 1 ], "axis1": [ 36, 2 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 425, 1 ], "axis1": [ 36, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 425, 1 ], "axis1": [ 37, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 36, 1 ], "axis1": [ 286, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 37, 1 ], "axis1": [ 286, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 286, 1 ], "matrix": "L18.mlp.W_out" } ], "l:26": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 46 ], "matrix": "L23.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 46 ], "matrix": "L23.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L23.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L23.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 33, 3 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 53, 4 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 166, 1 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 168, 4 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 50, 1 ], "axis1": [ 1, 46 ], "matrix": "L23.mlp.W_in" }, { "axis0": [ 1, 46 ], "axis1": [ 33, 3 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 1, 46 ], "axis1": [ 53, 4 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 1, 46 ], "axis1": [ 166, 1 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 1, 46 ], "axis1": [ 168, 4 ], "matrix": "L23.mlp.W_out" } ], "l:260": [ { "axis0": [ 0, 1 ], "axis1": [ 297, 1 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 297, 1 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 425, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 21, 3 ], "axis1": [ 297, 1 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 297, 1 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 297, 1 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 297, 1 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 297, 1 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 297, 1 ], "axis1": [ 425, 1 ], "matrix": "L17.mlp.W_out" } ], "l:261": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 28, 1 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 30, 3 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 63, 2 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 70, 6 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 153, 13 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 241, 8 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 296, 1 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 315, 2 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 318, 1 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 320, 3 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 328, 1 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 331, 8 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 349, 2 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 21, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 111, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 113, 5 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 120, 7 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" } ], "l:263": [ { "axis0": [ 0, 1 ], "axis1": [ 419, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 419, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 47, 1 ], "axis1": [ 419, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 479, 17 ], "axis1": [ 419, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 47, 1 ], "axis1": [ 419, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 479, 17 ], "axis1": [ 419, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 419, 17 ], "axis1": [ 28, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 419, 17 ], "axis1": [ 30, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 419, 17 ], "axis1": [ 153, 13 ], "matrix": "L15.mlp.W_out" } ], "l:265": [ { "axis0": [ 0, 1 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 479, 17 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 253, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 403, 1 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 403, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 404, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 405, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 406, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 407, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 408, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 409, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 410, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 411, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 412, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 413, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 414, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 415, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 416, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 417, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 418, 2 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 419, 1 ], "axis1": [ 217, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 217, 200 ], "axis1": [ 479, 17 ], "matrix": "L14.mlp.W_out" } ], "l:267": [ { "axis0": [ 0, 1 ], "axis1": [ 403, 17 ], "matrix": "L13.mlp.W_out" } ], "l:268": [ { "axis0": [ 0, 1 ], "axis1": [ 15, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 15, 200 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 253, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 26, 3 ], "axis1": [ 15, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 15, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 55, 4 ], "axis1": [ 15, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 61, 4 ], "axis1": [ 15, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 15, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 68, 8 ], "axis1": [ 15, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 15, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 15, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 298, 2 ], "axis1": [ 15, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 15, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 15, 200 ], "axis1": [ 253, 2 ], "matrix": "L13.mlp.W_out" } ], "l:27": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 205, 1 ], "axis1": [ 1, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 205, 1 ], "axis1": [ 2, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 1, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 2, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" } ], "l:270": [ { "axis0": [ 0, 1 ], "axis1": [ 13, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 13, 200 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 298, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 30, 3 ], "axis1": [ 13, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 76, 13 ], "axis1": [ 13, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 90, 3 ], "axis1": [ 13, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 153, 14 ], "axis1": [ 13, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 269, 2 ], "axis1": [ 13, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 13, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 13, 200 ], "axis1": [ 298, 2 ], "matrix": "L12.mlp.W_out" } ], "l:272": [ { "axis0": [ 0, 1 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 269, 2 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 19, 5 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 33, 3 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 93, 3 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 97, 4 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 249, 1 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 252, 1 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 255, 1 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 267, 2 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1101, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1101, 200 ], "axis1": [ 269, 2 ], "matrix": "L11.mlp.W_out" } ], "l:274": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L10.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 267, 2 ], "matrix": "L10.mlp.W_out" } ], "l:275": [ { "axis0": [ 0, 1 ], "axis1": [ 137, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 137, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 48, 1 ], "axis1": [ 137, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 314, 17 ], "axis1": [ 137, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 48, 1 ], "axis1": [ 137, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 314, 17 ], "axis1": [ 137, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 137, 17 ], "axis1": [ 241, 8 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 137, 17 ], "axis1": [ 296, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 137, 17 ], "axis1": [ 331, 8 ], "matrix": "L5.mlp.W_out" } ], "l:277": [ { "axis0": [ 0, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 23, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 26, 2 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 113, 3 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 117, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 23, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 26, 2 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 66, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 111, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 113, 3 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 117, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 120, 7 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 330, 1 ], "axis1": [ 547, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 547, 17 ], "axis1": [ 63, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 547, 17 ], "axis1": [ 70, 6 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 547, 17 ], "axis1": [ 315, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 547, 17 ], "axis1": [ 318, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 547, 17 ], "axis1": [ 320, 3 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 547, 17 ], "axis1": [ 328, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 547, 17 ], "axis1": [ 349, 2 ], "matrix": "L16.mlp.W_out" } ], "l:279": [ { "axis0": [ 0, 1 ], "axis1": [ 213, 206 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 213, 206 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 23, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 26, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 66, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 111, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 113, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 117, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 120, 7 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 330, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 350, 1 ], "axis1": [ 213, 206 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 213, 206 ], "axis1": [ 23, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 213, 206 ], "axis1": [ 26, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 213, 206 ], "axis1": [ 66, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 213, 206 ], "axis1": [ 111, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 213, 206 ], "axis1": [ 113, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 213, 206 ], "axis1": [ 117, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 213, 206 ], "axis1": [ 120, 7 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 213, 206 ], "axis1": [ 330, 1 ], "matrix": "L15.mlp.W_out" } ], "l:28": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L22.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L22.attn.W_K" }, { "axis0": [ 54, 3 ], "axis1": [ 0, 64 ], "matrix": "L22.attn.W_V" }, { "axis0": [ 168, 2 ], "axis1": [ 0, 64 ], "matrix": "L22.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 50, 1 ], "matrix": "L22.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 50, 1 ], "matrix": "L22.mlp.W_out" } ], "l:280": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 168, 1 ], "matrix": "L21.mlp.W_out" }, { "axis0": [ 166, 1 ], "axis1": [ 1, 2 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 168, 1 ], "matrix": "L21.mlp.W_out" } ], "l:281": [ { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 50, 1 ], "axis1": [ 9, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 53, 1 ], "axis1": [ 9, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 204, 1 ], "axis1": [ 9, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 286, 1 ], "axis1": [ 9, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 50, 1 ], "axis1": [ 10, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 53, 1 ], "axis1": [ 10, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 204, 1 ], "axis1": [ 10, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 286, 1 ], "axis1": [ 10, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 9, 1 ], "axis1": [ 166, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 10, 1 ], "axis1": [ 166, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 10, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 10, 1 ], "matrix": "L20.mlp.W_in" } ], "l:283": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_K" }, { "axis0": [ 202, 1 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 53, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 53, 1 ], "matrix": "L20.mlp.W_out" } ], "l:284": [ { "axis0": [ 0, 1 ], "axis1": [ 2, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 2, 1 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 202, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 37, 1 ], "axis1": [ 2, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 184, 3 ], "axis1": [ 2, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 188, 13 ], "axis1": [ 2, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 2, 1 ], "axis1": [ 202, 1 ], "matrix": "L19.mlp.W_out" } ], "l:285": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_K" }, { "axis0": [ 33, 3 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 38, 2 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 50, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 53, 3 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 67, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 166, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 168, 4 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 178, 3 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 245, 4 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 251, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 265, 3 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 269, 3 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 287, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 327, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 336, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 391, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 404, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 513, 17 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 37, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 184, 3 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 188, 13 ], "matrix": "L19.attn.W_O" } ], "l:287": [ { "axis0": [ 0, 1 ], "axis1": [ 623, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 623, 17 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 47, 1 ], "axis1": [ 623, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 51, 1 ], "axis1": [ 623, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 53, 2 ], "axis1": [ 623, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 256, 11 ], "axis1": [ 623, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 269, 3 ], "axis1": [ 623, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 47, 1 ], "axis1": [ 623, 17 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 51, 1 ], "axis1": [ 623, 17 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 53, 2 ], "axis1": [ 623, 17 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 256, 11 ], "axis1": [ 623, 17 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 269, 3 ], "axis1": [ 623, 17 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 623, 17 ], "axis1": [ 513, 17 ], "matrix": "L14.mlp.W_out" } ], "l:289": [ { "axis0": [ 0, 1 ], "axis1": [ 215, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 215, 200 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 51, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 53, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 11 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 269, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 26, 3 ], "axis1": [ 215, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 215, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 55, 4 ], "axis1": [ 215, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 61, 4 ], "axis1": [ 215, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 215, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 68, 8 ], "axis1": [ 215, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 215, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 215, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 298, 2 ], "axis1": [ 215, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 215, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 215, 200 ], "axis1": [ 51, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 215, 200 ], "axis1": [ 53, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 215, 200 ], "axis1": [ 256, 11 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 215, 200 ], "axis1": [ 269, 3 ], "matrix": "L13.mlp.W_out" } ], "l:290": [ { "axis0": [ 0, 1 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 26, 2 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 47, 3 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 51, 2 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 249, 2 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 252, 8 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 26, 2 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 47, 3 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 51, 2 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 66, 1 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 249, 2 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 252, 8 ], "axis1": [ 18, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 18, 17 ], "axis1": [ 53, 3 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 67, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 245, 4 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 265, 3 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 269, 3 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 287, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 336, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 391, 1 ], "matrix": "L18.mlp.W_out" } ], "l:292": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 200 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 26, 2 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 47, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 49, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 51, 2 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 66, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 249, 2 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 252, 8 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 106, 1 ], "axis1": [ 1, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 267, 1 ], "axis1": [ 1, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 445, 34 ], "axis1": [ 1, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1, 200 ], "axis1": [ 26, 2 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 47, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 49, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 51, 2 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 66, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 249, 2 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 252, 8 ], "matrix": "L17.mlp.W_out" } ], "l:294": [ { "axis0": [ 0, 1 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 26, 3 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 55, 4 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 61, 4 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 68, 8 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 251, 1 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 26, 3 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 52, 1 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 55, 4 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 61, 4 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 66, 1 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 68, 8 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 113, 5 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 120, 7 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 251, 1 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 330, 1 ], "axis1": [ 149, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 149, 34 ], "axis1": [ 445, 17 ], "matrix": "L14.mlp.W_out" } ], "l:296": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 251, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 250, 1 ], "axis1": [ 1, 2 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 251, 1 ], "matrix": "L13.mlp.W_out" } ], "l:297": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 59, 2 ], "axis1": [ 64, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 290, 3 ], "axis1": [ 64, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 297, 1 ], "axis1": [ 64, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 250, 1 ], "matrix": "L13.attn.W_O" } ], "l:299": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 250, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 290, 1 ], "matrix": "L12.mlp.W_out" } ], "l:30": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 4 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 4 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 56, 1 ], "matrix": "L20.mlp.W_out" }, { "axis0": [ 201, 1 ], "axis1": [ 5, 4 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 5, 4 ], "axis1": [ 56, 1 ], "matrix": "L20.mlp.W_out" } ], "l:300": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 26, 3 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 55, 4 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 61, 4 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 68, 8 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1, 100 ], "axis1": [ 250, 1 ], "matrix": "L11.mlp.W_out" } ], "l:302": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 291, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 250, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 3, 2 ], "axis1": [ 291, 1 ], "matrix": "L12.mlp.W_out" } ], "l:303": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 251, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 5, 2 ], "axis1": [ 292, 1 ], "matrix": "L12.mlp.W_out" } ], "l:304": [ { "axis0": [ 0, 1 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 30, 3 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 76, 13 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 90, 3 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 153, 14 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 101, 100 ], "axis1": [ 251, 1 ], "matrix": "L11.mlp.W_out" } ], "l:306": [ { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 60, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 251, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 7, 2 ], "axis1": [ 60, 1 ], "matrix": "L12.mlp.W_out" } ], "l:307": [ { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 50, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 9, 2 ], "axis1": [ 59, 1 ], "matrix": "L12.mlp.W_out" } ], "l:308": [ { "axis0": [ 0, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 19, 5 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 33, 3 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 93, 3 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 97, 4 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 249, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 252, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 255, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 201, 100 ], "axis1": [ 50, 1 ], "matrix": "L11.mlp.W_out" } ], "l:31": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 1 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 201, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 18, 1 ], "axis1": [ 1, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 26, 2 ], "axis1": [ 1, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 36, 1 ], "axis1": [ 1, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 1, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 76, 2 ], "axis1": [ 1, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 172, 6 ], "axis1": [ 1, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 181, 3 ], "axis1": [ 1, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 187, 1 ], "axis1": [ 1, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 1, 1 ], "axis1": [ 201, 1 ], "matrix": "L19.mlp.W_out" } ], "l:310": [ { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 297, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 50, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 11, 2 ], "axis1": [ 297, 1 ], "matrix": "L12.mlp.W_out" } ], "l:311": [ { "axis0": [ 0, 1 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 26, 3 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 55, 4 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 61, 4 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 68, 8 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 251, 1 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 26, 3 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 52, 1 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 55, 4 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 61, 4 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 66, 1 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 68, 8 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 113, 5 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 120, 7 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 251, 1 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 330, 1 ], "axis1": [ 183, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 183, 34 ], "axis1": [ 462, 17 ], "matrix": "L14.mlp.W_out" } ], "l:313": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 200 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 106, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 267, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 21, 1 ], "axis1": [ 1, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 1, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 116, 1 ], "axis1": [ 1, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 1, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 344, 5 ], "axis1": [ 1, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 420, 25 ], "axis1": [ 1, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1, 200 ], "axis1": [ 106, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 267, 1 ], "matrix": "L16.mlp.W_out" } ], "l:315": [ { "axis0": [ 0, 1 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 30, 3 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 76, 13 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 90, 3 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 153, 14 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 251, 1 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 30, 3 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 76, 13 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 90, 3 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 153, 14 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 251, 1 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 329, 1 ], "axis1": [ 81, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 81, 34 ], "axis1": [ 65, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 81, 34 ], "axis1": [ 324, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 81, 34 ], "axis1": [ 344, 5 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 81, 34 ], "axis1": [ 420, 8 ], "matrix": "L14.mlp.W_out" } ], "l:317": [ { "axis0": [ 0, 1 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 30, 3 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 76, 13 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 90, 3 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 153, 14 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 251, 1 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 30, 3 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 76, 13 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 90, 3 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 153, 14 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 251, 1 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 329, 1 ], "axis1": [ 115, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 115, 34 ], "axis1": [ 428, 17 ], "matrix": "L14.mlp.W_out" } ], "l:319": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 200 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 21, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 116, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 59, 2 ], "axis1": [ 1, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 1, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 1, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 267, 1 ], "axis1": [ 1, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 290, 3 ], "axis1": [ 1, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 297, 22 ], "axis1": [ 1, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 1, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 1, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 531, 2 ], "axis1": [ 1, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1, 200 ], "axis1": [ 21, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 116, 1 ], "matrix": "L15.mlp.W_out" } ], "l:32": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_K" }, { "axis0": [ 36, 2 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 42, 3 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 172, 6 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 181, 26 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 213, 1 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 215, 2 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 218, 4 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 227, 1 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 241, 4 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 334, 2 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 18, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 26, 2 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 36, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 66, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 76, 2 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 172, 6 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 181, 3 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 187, 1 ], "matrix": "L19.attn.W_O" } ], "l:321": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 19, 5 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 33, 3 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 93, 3 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 97, 4 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 249, 1 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 251, 2 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 255, 1 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 19, 5 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 33, 3 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 38, 2 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 49, 1 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 93, 3 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 97, 4 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 102, 4 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 111, 1 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 168, 4 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 178, 3 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 249, 1 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 251, 2 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 255, 1 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1, 34 ], "axis1": [ 59, 2 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 34 ], "axis1": [ 67, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 34 ], "axis1": [ 106, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 34 ], "axis1": [ 267, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 34 ], "axis1": [ 290, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 34 ], "axis1": [ 297, 9 ], "matrix": "L14.mlp.W_out" } ], "l:323": [ { "axis0": [ 0, 1 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 19, 5 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 33, 3 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 93, 3 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 97, 4 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 249, 1 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 251, 2 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 255, 1 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 19, 5 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 33, 3 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 38, 2 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 49, 1 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 93, 3 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 97, 4 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 102, 4 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 111, 1 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 168, 4 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 178, 3 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 249, 1 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 251, 2 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 255, 1 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 35, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 35, 34 ], "axis1": [ 306, 13 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 35, 34 ], "axis1": [ 320, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 35, 34 ], "axis1": [ 328, 1 ], "matrix": "L14.mlp.W_out" } ], "l:325": [ { "axis0": [ 0, 1 ], "axis1": [ 531, 2 ], "matrix": "L14.mlp.W_out" } ], "l:326": [ { "axis0": [ 0, 1 ], "axis1": [ 436, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 436, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 436, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 496, 17 ], "axis1": [ 436, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 436, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 496, 17 ], "axis1": [ 436, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 436, 17 ], "axis1": [ 33, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 436, 17 ], "axis1": [ 38, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 436, 17 ], "axis1": [ 50, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 436, 17 ], "axis1": [ 166, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 436, 17 ], "axis1": [ 168, 4 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 436, 17 ], "axis1": [ 178, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 436, 17 ], "axis1": [ 251, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 436, 17 ], "axis1": [ 327, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 436, 17 ], "axis1": [ 404, 1 ], "matrix": "L15.mlp.W_out" } ], "l:328": [ { "axis0": [ 0, 1 ], "axis1": [ 417, 206 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 417, 206 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 496, 17 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 349, 1 ], "axis1": [ 417, 206 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 417, 206 ], "axis1": [ 496, 17 ], "matrix": "L14.mlp.W_out" } ], "l:329": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 169, 1 ], "matrix": "L21.mlp.W_out" }, { "axis0": [ 33, 1 ], "axis1": [ 3, 2 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 3, 2 ], "axis1": [ 169, 1 ], "matrix": "L21.mlp.W_out" } ], "l:330": [ { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 50, 1 ], "axis1": [ 11, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 53, 1 ], "axis1": [ 11, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 203, 2 ], "axis1": [ 11, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 286, 1 ], "axis1": [ 11, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 50, 1 ], "axis1": [ 12, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 53, 1 ], "axis1": [ 12, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 203, 2 ], "axis1": [ 12, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 286, 1 ], "axis1": [ 12, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 11, 1 ], "axis1": [ 33, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 12, 1 ], "axis1": [ 33, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 11, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 12, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 11, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 12, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 33, 1 ], "matrix": "L20.mlp.W_out" } ], "l:332": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 1 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 203, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 32, 1 ], "axis1": [ 3, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 158, 8 ], "axis1": [ 3, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 207, 5 ], "axis1": [ 3, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 3, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 3, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 3, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 3, 1 ], "axis1": [ 203, 1 ], "matrix": "L19.mlp.W_out" } ], "l:333": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L18.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L18.attn.W_K" }, { "axis0": [ 68, 1 ], "axis1": [ 64, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 76, 7 ], "axis1": [ 64, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 357, 29 ], "axis1": [ 64, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 403, 1 ], "axis1": [ 64, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 405, 8 ], "axis1": [ 64, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 420, 5 ], "axis1": [ 64, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 32, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 158, 8 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 207, 5 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 214, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 333, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 392, 1 ], "matrix": "L18.attn.W_O" } ], "l:335": [ { "axis0": [ 0, 1 ], "axis1": [ 525, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 525, 17 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 47, 1 ], "axis1": [ 525, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 300, 17 ], "axis1": [ 525, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 47, 1 ], "axis1": [ 525, 17 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 300, 17 ], "axis1": [ 525, 17 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 525, 17 ], "axis1": [ 369, 17 ], "matrix": "L13.mlp.W_out" } ], "l:337": [ { "axis0": [ 0, 1 ], "axis1": [ 213, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 213, 200 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 300, 17 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 30, 3 ], "axis1": [ 213, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 76, 13 ], "axis1": [ 213, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 90, 3 ], "axis1": [ 213, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 153, 14 ], "axis1": [ 213, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 269, 2 ], "axis1": [ 213, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 213, 200 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 213, 200 ], "axis1": [ 300, 17 ], "matrix": "L12.mlp.W_out" } ], "l:338": [ { "axis0": [ 0, 1 ], "axis1": [ 280, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 280, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 48, 1 ], "axis1": [ 280, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 61, 2 ], "axis1": [ 280, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 298, 9 ], "axis1": [ 280, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 280, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 313, 2 ], "axis1": [ 280, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 48, 1 ], "axis1": [ 280, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 61, 2 ], "axis1": [ 280, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 298, 9 ], "axis1": [ 280, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 308, 4 ], "axis1": [ 280, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 313, 2 ], "axis1": [ 280, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 280, 17 ], "axis1": [ 357, 12 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 280, 17 ], "axis1": [ 420, 5 ], "matrix": "L17.mlp.W_out" } ], "l:34": [ { "axis0": [ 0, 1 ], "axis1": [ 69, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 47, 1 ], "axis1": [ 69, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 297, 17 ], "axis1": [ 69, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 47, 1 ], "axis1": [ 69, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 297, 17 ], "axis1": [ 69, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 69, 17 ], "axis1": [ 36, 2 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 69, 17 ], "axis1": [ 172, 6 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 69, 17 ], "axis1": [ 181, 9 ], "matrix": "L5.mlp.W_out" } ], "l:340": [ { "axis0": [ 0, 1 ], "axis1": [ 347, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 347, 200 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 61, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 298, 9 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 308, 4 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 313, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 21, 1 ], "axis1": [ 347, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 347, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 116, 1 ], "axis1": [ 347, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 347, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 344, 5 ], "axis1": [ 347, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 420, 25 ], "axis1": [ 347, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 347, 200 ], "axis1": [ 61, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 347, 200 ], "axis1": [ 298, 9 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 347, 200 ], "axis1": [ 308, 4 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 347, 200 ], "axis1": [ 313, 2 ], "matrix": "L16.mlp.W_out" } ], "l:341": [ { "axis0": [ 0, 1 ], "axis1": [ 564, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 564, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 564, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 272, 1 ], "axis1": [ 564, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 353, 16 ], "axis1": [ 564, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 564, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 272, 1 ], "axis1": [ 564, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 353, 16 ], "axis1": [ 564, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 564, 17 ], "axis1": [ 68, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 564, 17 ], "axis1": [ 76, 7 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 564, 17 ], "axis1": [ 403, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 564, 17 ], "axis1": [ 405, 8 ], "matrix": "L16.mlp.W_out" } ], "l:343": [ { "axis0": [ 0, 1 ], "axis1": [ 415, 110 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 415, 110 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 272, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 353, 16 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 267, 1 ], "axis1": [ 415, 110 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 415, 110 ], "axis1": [ 272, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 415, 110 ], "axis1": [ 353, 16 ], "matrix": "L13.mlp.W_out" } ], "l:344": [ { "axis0": [ 0, 1 ], "axis1": [ 6, 2 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 264, 1 ], "axis1": [ 6, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 264, 1 ], "axis1": [ 7, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 6, 1 ], "axis1": [ 205, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 7, 1 ], "axis1": [ 205, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 205, 1 ], "matrix": "L19.mlp.W_out" } ], "l:345": [ { "axis0": [ 0, 1 ], "axis1": [ 653, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 653, 2 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 264, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 48, 1 ], "axis1": [ 653, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 530, 1 ], "axis1": [ 653, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 653, 2 ], "axis1": [ 264, 1 ], "matrix": "L15.mlp.W_out" } ], "l:347": [ { "axis0": [ 0, 1 ], "axis1": [ 640, 2 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 2 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 530, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 251, 1 ], "axis1": [ 640, 2 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 640, 2 ], "axis1": [ 530, 1 ], "matrix": "L14.mlp.W_out" } ], "l:349": [ { "axis0": [ 0, 1 ], "axis1": [ 298, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 19, 1 ], "axis1": [ 298, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 83, 6 ], "axis1": [ 298, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 90, 6 ], "axis1": [ 298, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 298, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 298, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 386, 17 ], "axis1": [ 298, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 414, 1 ], "axis1": [ 298, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 416, 4 ], "axis1": [ 298, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 479, 10 ], "axis1": [ 298, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 490, 4 ], "axis1": [ 298, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 19, 1 ], "axis1": [ 315, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 83, 6 ], "axis1": [ 315, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 90, 6 ], "axis1": [ 315, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 315, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 315, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 386, 17 ], "axis1": [ 315, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 414, 1 ], "axis1": [ 315, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 416, 4 ], "axis1": [ 315, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 479, 10 ], "axis1": [ 315, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 490, 4 ], "axis1": [ 315, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 298, 17 ], "axis1": [ 426, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 315, 17 ], "axis1": [ 426, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" } ], "l:351": [ { "axis0": [ 0, 1 ], "axis1": [ 581, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 581, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 47, 1 ], "axis1": [ 581, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 273, 17 ], "axis1": [ 581, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 47, 1 ], "axis1": [ 581, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 273, 17 ], "axis1": [ 581, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 581, 17 ], "axis1": [ 83, 6 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 581, 17 ], "axis1": [ 90, 5 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 581, 17 ], "axis1": [ 329, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 581, 17 ], "axis1": [ 414, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 581, 17 ], "axis1": [ 416, 4 ], "matrix": "L16.mlp.W_out" } ], "l:353": [ { "axis0": [ 0, 1 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 273, 17 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 19, 5 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 33, 3 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 93, 3 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 97, 4 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 249, 1 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 252, 1 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 255, 1 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 267, 2 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1401, 200 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1401, 200 ], "axis1": [ 273, 17 ], "matrix": "L11.mlp.W_out" } ], "l:354": [ { "axis0": [ 0, 1 ], "axis1": [ 598, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 598, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 48, 2 ], "axis1": [ 598, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 51, 2 ], "axis1": [ 598, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 249, 2 ], "axis1": [ 598, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 252, 12 ], "axis1": [ 598, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 48, 2 ], "axis1": [ 598, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 51, 2 ], "axis1": [ 598, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 249, 2 ], "axis1": [ 598, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 252, 12 ], "axis1": [ 598, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 598, 17 ], "axis1": [ 19, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 598, 17 ], "axis1": [ 95, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 598, 17 ], "axis1": [ 97, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 598, 17 ], "axis1": [ 479, 10 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 598, 17 ], "axis1": [ 490, 4 ], "matrix": "L16.mlp.W_out" } ], "l:356": [ { "axis0": [ 0, 1 ], "axis1": [ 453, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 453, 200 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 49, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 51, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 249, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 252, 12 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 59, 2 ], "axis1": [ 453, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 453, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 453, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 267, 1 ], "axis1": [ 453, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 290, 3 ], "axis1": [ 453, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 297, 22 ], "axis1": [ 453, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 453, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 453, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 531, 2 ], "axis1": [ 453, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 453, 200 ], "axis1": [ 49, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 453, 200 ], "axis1": [ 51, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 453, 200 ], "axis1": [ 249, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 453, 200 ], "axis1": [ 252, 12 ], "matrix": "L15.mlp.W_out" } ], "l:357": [ { "axis0": [ 0, 1 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 317, 2 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 320, 7 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 344, 5 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 65, 1 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 67, 1 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 317, 2 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 320, 7 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 328, 1 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 344, 5 ], "axis1": [ 542, 17 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 542, 17 ], "axis1": [ 386, 17 ], "matrix": "L13.mlp.W_out" } ], "l:359": [ { "axis0": [ 0, 1 ], "axis1": [ 413, 38 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 413, 38 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 65, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 67, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 317, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 7 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 328, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 344, 5 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 268, 1 ], "axis1": [ 413, 38 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 413, 38 ], "axis1": [ 65, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 413, 38 ], "axis1": [ 67, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 413, 38 ], "axis1": [ 317, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 413, 38 ], "axis1": [ 320, 7 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 413, 38 ], "axis1": [ 328, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 413, 38 ], "axis1": [ 344, 5 ], "matrix": "L12.mlp.W_out" } ], "l:36": [ { "axis0": [ 22, 1 ], "axis1": [ 192, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 117, 1 ], "axis1": [ 192, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 33, 1 ], "axis1": [ 192, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 120, 1 ], "axis1": [ 192, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 18, 1 ], "axis1": [ 192, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 47, 1 ], "matrix": "L4.attn.W_O" } ], "l:360": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 51, 1 ], "axis1": [ 64, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 258, 1 ], "axis1": [ 64, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 268, 1 ], "matrix": "L12.attn.W_O" } ], "l:361": [ { "axis0": [ 0, 1 ], "axis1": [ 38, 2 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 42, 1 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 178, 3 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 201, 6 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 213, 1 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 215, 1 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 327, 1 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 334, 1 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 404, 1 ], "matrix": "L23.mlp.W_out" } ], "l:362": [ { "axis0": [ 0, 1 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 18, 2 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 21, 2 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 90, 6 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 106, 3 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 110, 4 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 340, 1 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 18, 2 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 21, 2 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 90, 6 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 106, 3 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 110, 4 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 340, 1 ], "axis1": [ 18, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 18, 17 ], "axis1": [ 42, 1 ], "matrix": "L25.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 53, 4 ], "matrix": "L25.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 76, 2 ], "matrix": "L25.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 211, 1 ], "matrix": "L25.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 213, 3 ], "matrix": "L25.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 334, 1 ], "matrix": "L25.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 1895, 5 ], "matrix": "L25.mlp.W_out" } ], "l:364": [ { "axis0": [ 0, 1 ], "axis1": [ 156, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 156, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 340, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 285, 2 ], "axis1": [ 156, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 156, 2 ], "axis1": [ 340, 1 ], "matrix": "L5.mlp.W_out" } ], "l:366": [ { "axis0": [ 0, 1 ], "axis1": [ 166, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 166, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 285, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 166, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 166, 2 ], "axis1": [ 285, 1 ], "matrix": "L4.mlp.W_out" } ], "l:367": [ { "axis0": [ 0, 1 ], "axis1": [ 168, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 168, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 286, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 168, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 168, 2 ], "axis1": [ 286, 1 ], "matrix": "L4.mlp.W_out" } ], "l:368": [ { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 20, 1 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 69, 1 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 97, 4 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 102, 4 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 344, 2 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 458, 6 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 465, 14 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 480, 9 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 490, 4 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 495, 4 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 500, 4 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 505, 1 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 507, 6 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 515, 4 ], "axis1": [ 256, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 18, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 21, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 90, 6 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 106, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 110, 4 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 520, 1 ], "axis1": [ 320, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 530, 1 ], "axis1": [ 320, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 533, 62 ], "axis1": [ 320, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 18, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 21, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 90, 6 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 106, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 110, 4 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 595, 64 ], "axis1": [ 384, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 18, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 21, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 90, 6 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 106, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 110, 4 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 659, 12 ], "axis1": [ 448, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 18, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 21, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 90, 6 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 106, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 110, 4 ], "matrix": "L25.attn.W_O" } ], "l:369": [ { "axis0": [ 0, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 18, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 19, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 21, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 21, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 22, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 23, 6 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 30, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 30, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 31, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 32, 6 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 37, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 76, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 90, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 91, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 92, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 93, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 94, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 106, 3 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 111, 3 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 113, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 114, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 115, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 116, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 117, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 120, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 120, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 121, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 122, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 123, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 124, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 125, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 126, 10 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 153, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 153, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 154, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 155, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 156, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 157, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 158, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 159, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 160, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 161, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 162, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 163, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 164, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 165, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 168, 10 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 181, 4 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 184, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 185, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 186, 3 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 188, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 189, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 190, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 191, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 192, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 193, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 194, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 195, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 196, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 197, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 198, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 199, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 200, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 207, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 207, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 208, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 209, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 210, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 330, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 426, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 426, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 427, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 428, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 429, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 430, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 431, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 432, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 433, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 434, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 435, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 436, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 437, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 438, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 439, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 440, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 441, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 442, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 18, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 19, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 21, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 21, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 22, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 23, 6 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 28, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 30, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 30, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 31, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 32, 6 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 37, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 40, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 53, 4 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 66, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 76, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 90, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 91, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 92, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 93, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 94, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 106, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 106, 3 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 110, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 111, 3 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 113, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 114, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 115, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 116, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 117, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 120, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 120, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 121, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 122, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 123, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 124, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 125, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 126, 10 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 153, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 153, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 154, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 155, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 156, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 157, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 158, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 159, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 160, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 161, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 162, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 163, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 164, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 165, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 168, 10 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 181, 4 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 184, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 185, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 186, 3 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 188, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 189, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 190, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 191, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 192, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 193, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 194, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 195, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 196, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 197, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 198, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 199, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 200, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 207, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 207, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 208, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 209, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 210, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 211, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 214, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 214, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 330, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 330, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 331, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 426, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 426, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 427, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 428, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 429, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 430, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 431, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 432, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 433, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 434, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 435, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 436, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 437, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 438, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 439, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 440, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 441, 2 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 442, 1 ], "axis1": [ 205, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 205, 204 ], "axis1": [ 20, 1 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 69, 1 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 97, 4 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 102, 4 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 344, 2 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 458, 6 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 465, 14 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 480, 9 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 490, 4 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 495, 4 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 500, 4 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 505, 1 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 507, 6 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 515, 4 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 520, 1 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 530, 1 ], "matrix": "L24.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 533, 138 ], "matrix": "L24.mlp.W_out" } ], "l:372": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 181, 51 ], "axis1": [ 128, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 24, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 40, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 107, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 110, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 112, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 127, 9 ], "matrix": "L5.attn.W_O" } ], "l:374": [ { "axis0": [ 0, 1 ], "axis1": [ 60, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 60, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 47, 1 ], "axis1": [ 60, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 127, 17 ], "axis1": [ 60, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 47, 1 ], "axis1": [ 60, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 127, 17 ], "axis1": [ 60, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 60, 17 ], "axis1": [ 181, 17 ], "matrix": "L4.mlp.W_out" } ], "l:376": [ { "axis0": [ 0, 1 ], "axis1": [ 127, 17 ], "matrix": "L3.mlp.W_out" } ], "l:377": [ { "axis0": [ 0, 1 ], "axis1": [ 77, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 77, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 48, 1 ], "axis1": [ 77, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 144, 17 ], "axis1": [ 77, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 48, 1 ], "axis1": [ 77, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 144, 17 ], "axis1": [ 77, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 77, 17 ], "axis1": [ 198, 17 ], "matrix": "L4.mlp.W_out" } ], "l:379": [ { "axis0": [ 0, 1 ], "axis1": [ 144, 17 ], "matrix": "L3.mlp.W_out" } ], "l:38": [ { "axis0": [ 0, 1 ], "axis1": [ 117, 1 ], "matrix": "L3.mlp.W_out" } ], "l:380": [ { "axis0": [ 0, 1 ], "axis1": [ 94, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 94, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 94, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 161, 17 ], "axis1": [ 94, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 94, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 161, 17 ], "axis1": [ 94, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 94, 17 ], "axis1": [ 215, 17 ], "matrix": "L4.mlp.W_out" } ], "l:382": [ { "axis0": [ 0, 1 ], "axis1": [ 161, 17 ], "matrix": "L3.mlp.W_out" } ], "l:383": [ { "axis0": [ 0, 1 ], "axis1": [ 35, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 23, 3 ], "axis1": [ 35, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 114, 4 ], "axis1": [ 35, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 120, 9 ], "axis1": [ 35, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 293, 1 ], "axis1": [ 35, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 35, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 23, 3 ], "axis1": [ 35, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 114, 4 ], "axis1": [ 35, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 120, 9 ], "axis1": [ 35, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 293, 1 ], "axis1": [ 35, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 330, 1 ], "axis1": [ 35, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 35, 17 ], "axis1": [ 1900, 17 ], "matrix": "L25.mlp.W_out" } ], "l:385": [ { "axis0": [ 0, 1 ], "axis1": [ 182, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 182, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 293, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 38, 2 ], "axis1": [ 182, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 182, 2 ], "axis1": [ 293, 1 ], "matrix": "L4.mlp.W_out" } ], "l:387": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 38, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 3, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 3, 2 ], "axis1": [ 38, 1 ], "matrix": "L3.mlp.W_out" } ], "l:388": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 39, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 5, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 5, 2 ], "axis1": [ 39, 1 ], "matrix": "L3.mlp.W_out" } ], "l:389": [ { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 671, 64 ], "axis1": [ 512, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 23, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 114, 4 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 120, 9 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 330, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 735, 64 ], "axis1": [ 576, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 23, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 114, 4 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 120, 9 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 330, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 799, 64 ], "axis1": [ 640, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 23, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 114, 4 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 120, 9 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 330, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 863, 12 ], "axis1": [ 704, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 23, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 114, 4 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 120, 9 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 330, 1 ], "matrix": "L25.attn.W_O" } ], "l:39": [ { "axis0": [ 0, 1 ], "axis1": [ 22, 1 ], "matrix": "L3.mlp.W_out" } ], "l:390": [ { "axis0": [ 0, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 19, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 21, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 21, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 22, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 23, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 24, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 25, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 30, 3 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 32, 4 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 37, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 37, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 90, 6 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 106, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 107, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 108, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 111, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 112, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 113, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 114, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 115, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 116, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 117, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 120, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 120, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 121, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 122, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 123, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 124, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 125, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 126, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 127, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 128, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 129, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 130, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 131, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 132, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 133, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 134, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 135, 24 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 158, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 159, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 160, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 161, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 162, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 163, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 164, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 165, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 184, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 184, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 185, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 186, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 188, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 188, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 189, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 190, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 191, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 192, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 193, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 194, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 195, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 196, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 197, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 198, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 199, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 200, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 207, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 207, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 208, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 209, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 210, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 330, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 426, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 426, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 427, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 428, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 429, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 430, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 431, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 432, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 433, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 434, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 435, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 436, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 437, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 438, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 439, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 440, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 441, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 442, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 19, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 21, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 21, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 22, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 23, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 24, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 25, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 28, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 30, 3 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 32, 4 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 37, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 37, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 40, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 41, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 53, 4 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 90, 6 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 106, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 107, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 108, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 110, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 110, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 111, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 112, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 113, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 114, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 115, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 116, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 117, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 120, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 120, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 121, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 122, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 123, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 124, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 125, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 126, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 127, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 128, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 129, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 130, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 131, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 132, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 133, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 134, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 135, 24 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 158, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 159, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 160, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 161, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 162, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 163, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 164, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 165, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 168, 4 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 184, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 184, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 185, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 186, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 188, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 188, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 189, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 190, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 191, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 192, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 193, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 194, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 195, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 196, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 197, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 198, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 199, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 200, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 207, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 207, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 208, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 209, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 210, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 211, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 214, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 214, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 330, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 330, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 426, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 426, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 427, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 428, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 429, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 430, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 431, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 432, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 433, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 434, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 435, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 436, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 437, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 438, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 439, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 440, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 441, 2 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 442, 1 ], "axis1": [ 409, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 409, 204 ], "axis1": [ 671, 204 ], "matrix": "L24.mlp.W_out" } ], "l:393": [ { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 232, 51 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 136, 17 ], "matrix": "L5.attn.W_O" } ], "l:395": [ { "axis0": [ 0, 1 ], "axis1": [ 111, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 111, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 47, 1 ], "axis1": [ 111, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 127, 17 ], "axis1": [ 111, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 47, 1 ], "axis1": [ 111, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 127, 17 ], "axis1": [ 111, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 111, 17 ], "axis1": [ 232, 17 ], "matrix": "L4.mlp.W_out" } ], "l:397": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 48, 1 ], "axis1": [ 128, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 144, 17 ], "axis1": [ 128, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 48, 1 ], "axis1": [ 128, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 144, 17 ], "axis1": [ 128, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 128, 17 ], "axis1": [ 249, 17 ], "matrix": "L4.mlp.W_out" } ], "l:399": [ { "axis0": [ 0, 1 ], "axis1": [ 145, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 145, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 145, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 161, 17 ], "axis1": [ 145, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 145, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 161, 17 ], "axis1": [ 145, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 145, 17 ], "axis1": [ 266, 17 ], "matrix": "L4.mlp.W_out" } ], "l:4": [ { "axis0": [ 0, 1 ], "axis1": [ 107, 1 ], "matrix": "L0.mlp.W_out" } ], "l:401": [ { "axis0": [ 0, 1 ], "axis1": [ 52, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 52, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 26, 3 ], "axis1": [ 52, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 52, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 129, 12 ], "axis1": [ 52, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 52, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 341, 1 ], "axis1": [ 52, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 26, 3 ], "axis1": [ 52, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 66, 1 ], "axis1": [ 52, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 129, 12 ], "axis1": [ 52, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 331, 1 ], "axis1": [ 52, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 341, 1 ], "axis1": [ 52, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 52, 17 ], "axis1": [ 1917, 17 ], "matrix": "L25.mlp.W_out" } ], "l:403": [ { "axis0": [ 0, 1 ], "axis1": [ 158, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 158, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 341, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 287, 2 ], "axis1": [ 158, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 158, 2 ], "axis1": [ 341, 1 ], "matrix": "L5.mlp.W_out" } ], "l:405": [ { "axis0": [ 0, 1 ], "axis1": [ 170, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 170, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 287, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 170, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 170, 2 ], "axis1": [ 287, 1 ], "matrix": "L4.mlp.W_out" } ], "l:406": [ { "axis0": [ 0, 1 ], "axis1": [ 172, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 172, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 288, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 172, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 172, 2 ], "axis1": [ 288, 1 ], "matrix": "L4.mlp.W_out" } ], "l:407": [ { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 875, 64 ], "axis1": [ 768, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 26, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 66, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 129, 12 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 331, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 939, 64 ], "axis1": [ 832, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 26, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 66, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 129, 12 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 331, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1003, 64 ], "axis1": [ 896, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 26, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 66, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 129, 12 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 331, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1067, 12 ], "axis1": [ 960, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 960, 64 ], "axis1": [ 26, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 960, 64 ], "axis1": [ 66, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 960, 64 ], "axis1": [ 129, 12 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 960, 64 ], "axis1": [ 331, 1 ], "matrix": "L25.attn.W_O" } ], "l:408": [ { "axis0": [ 0, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 21, 4 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 24, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 25, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 32, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 32, 4 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 37, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 37, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 107, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 107, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 108, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 3 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 112, 6 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 120, 8 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 127, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 128, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 129, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 130, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 131, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 132, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 133, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 134, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 135, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 158, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 158, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 159, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 160, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 161, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 162, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 163, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 164, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 165, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 184, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 184, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 185, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 186, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 188, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 188, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 189, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 190, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 191, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 192, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 193, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 194, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 195, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 196, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 197, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 198, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 199, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 200, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 207, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 207, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 208, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 209, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 210, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 426, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 426, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 427, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 428, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 429, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 430, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 431, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 432, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 433, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 434, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 435, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 436, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 437, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 438, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 439, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 440, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 441, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 442, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 21, 4 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 24, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 25, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 32, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 32, 4 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 37, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 37, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 40, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 41, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 53, 4 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 107, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 107, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 108, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 110, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 110, 3 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 112, 6 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 120, 8 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 127, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 128, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 129, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 130, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 131, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 132, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 133, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 134, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 135, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 158, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 158, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 159, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 160, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 161, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 162, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 163, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 164, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 165, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 168, 4 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 184, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 184, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 185, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 186, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 188, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 188, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 189, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 190, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 191, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 192, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 193, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 194, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 195, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 196, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 197, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 198, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 199, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 200, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 207, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 207, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 208, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 209, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 210, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 211, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 214, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 214, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 330, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 426, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 426, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 427, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 428, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 429, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 430, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 431, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 432, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 433, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 434, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 435, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 436, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 437, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 438, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 439, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 440, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 441, 2 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 442, 1 ], "axis1": [ 613, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 613, 204 ], "axis1": [ 875, 204 ], "matrix": "L24.mlp.W_out" } ], "l:41": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 33, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 26, 1 ], "axis1": [ 1, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 1, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 33, 1 ], "matrix": "L3.mlp.W_out" } ], "l:411": [ { "axis0": [ 0, 1 ], "axis1": [ 69, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 30, 2 ], "axis1": [ 69, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 141, 15 ], "axis1": [ 69, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 294, 1 ], "axis1": [ 69, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 30, 2 ], "axis1": [ 69, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 141, 15 ], "axis1": [ 69, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 294, 1 ], "axis1": [ 69, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 69, 17 ], "axis1": [ 1934, 17 ], "matrix": "L25.mlp.W_out" } ], "l:413": [ { "axis0": [ 0, 1 ], "axis1": [ 184, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 184, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 294, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 40, 2 ], "axis1": [ 184, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 184, 2 ], "axis1": [ 294, 1 ], "matrix": "L4.mlp.W_out" } ], "l:415": [ { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 40, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 7, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 7, 2 ], "axis1": [ 40, 1 ], "matrix": "L3.mlp.W_out" } ], "l:416": [ { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 41, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 9, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 9, 2 ], "axis1": [ 41, 1 ], "matrix": "L3.mlp.W_out" } ], "l:417": [ { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1079, 64 ], "axis1": [ 1024, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1024, 64 ], "axis1": [ 30, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 141, 15 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1143, 64 ], "axis1": [ 1088, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1088, 64 ], "axis1": [ 30, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1088, 64 ], "axis1": [ 141, 15 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1207, 64 ], "axis1": [ 1152, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1152, 64 ], "axis1": [ 30, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 141, 15 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1271, 12 ], "axis1": [ 1216, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1216, 64 ], "axis1": [ 30, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 141, 15 ], "matrix": "L25.attn.W_O" } ], "l:418": [ { "axis0": [ 0, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 24, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 24, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 25, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 32, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 32, 4 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 37, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 107, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 107, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 108, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 112, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 112, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 127, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 127, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 128, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 129, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 130, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 131, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 132, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 133, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 134, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 135, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 158, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 158, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 159, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 160, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 161, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 162, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 163, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 164, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 165, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 184, 3 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 188, 13 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 207, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 207, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 208, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 209, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 210, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 426, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 426, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 427, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 428, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 429, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 430, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 431, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 432, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 433, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 434, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 435, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 436, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 437, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 438, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 439, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 440, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 441, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 442, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 24, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 24, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 25, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 32, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 32, 4 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 37, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 40, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 41, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 53, 4 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 107, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 107, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 108, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 110, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 110, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 112, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 112, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 127, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 127, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 128, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 129, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 130, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 131, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 132, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 133, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 134, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 135, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 158, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 158, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 159, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 160, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 161, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 162, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 163, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 164, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 165, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 168, 4 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 184, 3 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 188, 13 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 207, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 207, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 208, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 209, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 210, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 211, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 214, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 214, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 426, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 426, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 427, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 428, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 429, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 430, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 431, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 432, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 433, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 434, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 435, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 436, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 437, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 438, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 439, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 440, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 441, 2 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 442, 1 ], "axis1": [ 817, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 817, 204 ], "axis1": [ 1079, 204 ], "matrix": "L24.mlp.W_out" } ], "l:421": [ { "axis0": [ 0, 1 ], "axis1": [ 86, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 86, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 32, 2 ], "axis1": [ 86, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 156, 11 ], "axis1": [ 86, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 86, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 342, 1 ], "axis1": [ 86, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 32, 2 ], "axis1": [ 86, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 156, 11 ], "axis1": [ 86, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 168, 4 ], "axis1": [ 86, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 342, 1 ], "axis1": [ 86, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 86, 17 ], "axis1": [ 1951, 17 ], "matrix": "L25.mlp.W_out" } ], "l:423": [ { "axis0": [ 0, 1 ], "axis1": [ 160, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 160, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 342, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 289, 2 ], "axis1": [ 160, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 160, 2 ], "axis1": [ 342, 1 ], "matrix": "L5.mlp.W_out" } ], "l:425": [ { "axis0": [ 0, 1 ], "axis1": [ 174, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 174, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 289, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 174, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 174, 2 ], "axis1": [ 289, 1 ], "matrix": "L4.mlp.W_out" } ], "l:426": [ { "axis0": [ 0, 1 ], "axis1": [ 176, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 176, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 290, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 176, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 176, 2 ], "axis1": [ 290, 1 ], "matrix": "L4.mlp.W_out" } ], "l:427": [ { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1283, 64 ], "axis1": [ 1280, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1280, 64 ], "axis1": [ 32, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 156, 11 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 168, 4 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1347, 64 ], "axis1": [ 1344, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1344, 64 ], "axis1": [ 32, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 156, 11 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 168, 4 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1411, 64 ], "axis1": [ 1408, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1408, 64 ], "axis1": [ 32, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1408, 64 ], "axis1": [ 156, 11 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1408, 64 ], "axis1": [ 168, 4 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1475, 12 ], "axis1": [ 1472, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1472, 64 ], "axis1": [ 32, 2 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 156, 11 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 168, 4 ], "matrix": "L25.attn.W_O" } ], "l:428": [ { "axis0": [ 0, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 24, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 24, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 25, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 32, 4 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 107, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 107, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 108, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 112, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 112, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 127, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 127, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 128, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 129, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 130, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 131, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 132, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 133, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 134, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 135, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 158, 9 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 207, 5 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 426, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 426, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 427, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 428, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 429, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 430, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 431, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 432, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 433, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 434, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 435, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 436, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 437, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 438, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 439, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 440, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 441, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 442, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 24, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 24, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 25, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 32, 4 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 40, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 41, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 53, 4 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 107, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 107, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 108, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 110, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 110, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 112, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 112, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 127, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 127, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 128, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 129, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 130, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 131, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 132, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 133, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 134, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 135, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 158, 9 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 168, 4 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 207, 5 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 214, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 426, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 426, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 427, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 428, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 429, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 430, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 431, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 432, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 433, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 434, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 435, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 436, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 437, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 438, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 439, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 440, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 441, 2 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 442, 1 ], "axis1": [ 1021, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 1021, 204 ], "axis1": [ 1283, 204 ], "matrix": "L24.mlp.W_out" } ], "l:43": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 26, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 107, 2 ], "axis1": [ 1, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 1, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 26, 1 ], "matrix": "L2.mlp.W_out" } ], "l:431": [ { "axis0": [ 0, 1 ], "axis1": [ 103, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 103, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 34, 3 ], "axis1": [ 103, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 172, 13 ], "axis1": [ 103, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 295, 1 ], "axis1": [ 103, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 103, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 34, 3 ], "axis1": [ 103, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 172, 13 ], "axis1": [ 103, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 295, 1 ], "axis1": [ 103, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 103, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 103, 17 ], "axis1": [ 1968, 17 ], "matrix": "L25.mlp.W_out" } ], "l:433": [ { "axis0": [ 0, 1 ], "axis1": [ 186, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 186, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 295, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 107, 1 ], "axis1": [ 186, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 186, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 186, 2 ], "axis1": [ 295, 1 ], "matrix": "L4.mlp.W_out" } ], "l:435": [ { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 110, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 11, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 11, 2 ], "axis1": [ 110, 1 ], "matrix": "L3.mlp.W_out" } ], "l:436": [ { "axis0": [ 0, 1 ], "axis1": [ 13, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 13, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 13, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 107, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 13, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 13, 2 ], "axis1": [ 107, 1 ], "matrix": "L3.mlp.W_out" } ], "l:437": [ { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1487, 64 ], "axis1": [ 1536, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1536, 64 ], "axis1": [ 34, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 172, 13 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 392, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1551, 64 ], "axis1": [ 1600, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1600, 64 ], "axis1": [ 34, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 172, 13 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 392, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1615, 64 ], "axis1": [ 1664, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1664, 64 ], "axis1": [ 34, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 172, 13 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 392, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1728, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1728, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1679, 12 ], "axis1": [ 1728, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1728, 64 ], "axis1": [ 34, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 172, 13 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 392, 1 ], "matrix": "L25.attn.W_O" } ], "l:438": [ { "axis0": [ 0, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 24, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 24, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 25, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 33, 3 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 107, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 107, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 108, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 112, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 112, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 127, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 127, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 128, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 129, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 130, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 131, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 132, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 133, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 134, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 135, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 166, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 426, 17 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 24, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 24, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 25, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 33, 3 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 40, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 41, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 53, 4 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 107, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 107, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 108, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 110, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 110, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 112, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 112, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 127, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 127, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 128, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 129, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 130, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 131, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 132, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 133, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 134, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 135, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 2 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 166, 1 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 168, 4 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 426, 17 ], "axis1": [ 1225, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 1225, 204 ], "axis1": [ 1487, 204 ], "matrix": "L24.mlp.W_out" } ], "l:441": [ { "axis0": [ 0, 1 ], "axis1": [ 120, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 120, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 37, 3 ], "axis1": [ 120, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 185, 12 ], "axis1": [ 120, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 120, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 120, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 343, 1 ], "axis1": [ 120, 17 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 37, 3 ], "axis1": [ 120, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 185, 12 ], "axis1": [ 120, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 120, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 120, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 343, 1 ], "axis1": [ 120, 17 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 120, 17 ], "axis1": [ 1985, 17 ], "matrix": "L25.mlp.W_out" } ], "l:443": [ { "axis0": [ 0, 1 ], "axis1": [ 162, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 162, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 343, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 291, 2 ], "axis1": [ 162, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 162, 2 ], "axis1": [ 343, 1 ], "matrix": "L5.mlp.W_out" } ], "l:445": [ { "axis0": [ 0, 1 ], "axis1": [ 178, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 178, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 291, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 178, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 178, 2 ], "axis1": [ 291, 1 ], "matrix": "L4.mlp.W_out" } ], "l:446": [ { "axis0": [ 0, 1 ], "axis1": [ 180, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 180, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 292, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 180, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 180, 2 ], "axis1": [ 292, 1 ], "matrix": "L4.mlp.W_out" } ], "l:447": [ { "axis0": [ 0, 1 ], "axis1": [ 1792, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1792, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1691, 64 ], "axis1": [ 1792, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1792, 64 ], "axis1": [ 37, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 185, 12 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 327, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 333, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1856, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1856, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1755, 64 ], "axis1": [ 1856, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1856, 64 ], "axis1": [ 37, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 185, 12 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 327, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 333, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1920, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1920, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1819, 64 ], "axis1": [ 1920, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1920, 64 ], "axis1": [ 37, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 185, 12 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 327, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 333, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1984, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1984, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 1883, 12 ], "axis1": [ 1984, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 1984, 64 ], "axis1": [ 37, 3 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1984, 64 ], "axis1": [ 185, 12 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1984, 64 ], "axis1": [ 327, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 1984, 64 ], "axis1": [ 333, 1 ], "matrix": "L25.attn.W_O" } ], "l:448": [ { "axis0": [ 0, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 24, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 33, 3 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 107, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 112, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 127, 10 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 138, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 139, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 140, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 141, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 142, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 143, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 144, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 145, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 146, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 147, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 148, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 149, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 150, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 151, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 166, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 24, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 33, 3 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 40, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 53, 4 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 107, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 110, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 112, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 127, 10 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 137, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 138, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 139, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 140, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 141, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 142, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 143, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 144, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 145, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 146, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 147, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 148, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 149, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 150, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 151, 2 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 166, 1 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 168, 4 ], "axis1": [ 1429, 204 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 1429, 204 ], "axis1": [ 1691, 204 ], "matrix": "L24.mlp.W_out" } ], "l:45": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 1, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 108, 1 ], "matrix": "L1.mlp.W_out" } ], "l:451": [ { "axis0": [ 0, 1 ], "axis1": [ 57, 4 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 285, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 288, 5 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 296, 5 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 337, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 393, 1 ], "matrix": "L18.mlp.W_out" } ], "l:46": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 18, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 1, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 1, 1 ], "axis1": [ 18, 1 ], "matrix": "L0.mlp.W_out" } ], "l:47": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 19, 1 ], "axis1": [ 3, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 3, 2 ], "axis1": [ 107, 1 ], "matrix": "L1.mlp.W_out" } ], "l:48": [ { "axis0": [ 0, 1 ], "axis1": [ 2, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 2, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 19, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 2, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 2, 1 ], "axis1": [ 19, 1 ], "matrix": "L0.mlp.W_out" } ], "l:49": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 20, 1 ], "axis1": [ 5, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 5, 2 ], "axis1": [ 110, 1 ], "matrix": "L1.mlp.W_out" } ], "l:5": [ { "axis0": [ 0, 1 ], "axis1": [ 108, 1 ], "matrix": "L0.mlp.W_out" } ], "l:50": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 20, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 3, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 3, 1 ], "axis1": [ 20, 1 ], "matrix": "L0.mlp.W_out" } ], "l:51": [ { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 111, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 27, 1 ], "axis1": [ 7, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 7, 2 ], "axis1": [ 111, 1 ], "matrix": "L1.mlp.W_out" } ], "l:52": [ { "axis0": [ 24, 2 ], "axis1": [ 0, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 0, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 26, 1 ], "axis1": [ 0, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 21, 1 ], "axis1": [ 0, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 27, 1 ], "matrix": "L1.attn.W_O" } ], "l:54": [ { "axis0": [ 0, 1 ], "axis1": [ 24, 1 ], "matrix": "L0.mlp.W_out" } ], "l:55": [ { "axis0": [ 0, 1 ], "axis1": [ 25, 1 ], "matrix": "L0.mlp.W_out" } ], "l:57": [ { "axis0": [ 0, 1 ], "axis1": [ 26, 1 ], "matrix": "L0.mlp.W_out" } ], "l:58": [ { "axis0": [ 0, 1 ], "axis1": [ 120, 1 ], "matrix": "L3.mlp.W_out" } ], "l:59": [ { "axis0": [ 0, 1 ], "axis1": [ 297, 17 ], "matrix": "L4.mlp.W_out" } ], "l:60": [ { "axis0": [ 0, 1 ], "axis1": [ 86, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 86, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 48, 1 ], "axis1": [ 86, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 314, 17 ], "axis1": [ 86, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 48, 1 ], "axis1": [ 86, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 314, 17 ], "axis1": [ 86, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 86, 17 ], "axis1": [ 190, 17 ], "matrix": "L5.mlp.W_out" } ], "l:62": [ { "axis0": [ 121, 2 ], "axis1": [ 256, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 33, 1 ], "axis1": [ 256, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 123, 1 ], "axis1": [ 256, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 19, 1 ], "axis1": [ 256, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 48, 1 ], "matrix": "L4.attn.W_O" } ], "l:64": [ { "axis0": [ 0, 1 ], "axis1": [ 121, 1 ], "matrix": "L3.mlp.W_out" } ], "l:65": [ { "axis0": [ 0, 1 ], "axis1": [ 122, 1 ], "matrix": "L3.mlp.W_out" } ], "l:67": [ { "axis0": [ 0, 1 ], "axis1": [ 123, 1 ], "matrix": "L3.mlp.W_out" } ], "l:68": [ { "axis0": [ 0, 1 ], "axis1": [ 314, 17 ], "matrix": "L4.mlp.W_out" } ], "l:69": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 1, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 260, 4 ], "axis1": [ 1, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 272, 13 ], "axis1": [ 1, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 1, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 260, 4 ], "axis1": [ 1, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 272, 13 ], "axis1": [ 1, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 1, 17 ], "axis1": [ 42, 3 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 213, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 215, 2 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 218, 4 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 227, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 241, 4 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 334, 2 ], "matrix": "L18.mlp.W_out" } ], "l:7": [ { "axis0": [ 0, 1 ], "axis1": [ 4, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 4, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 21, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 4, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 4, 1 ], "axis1": [ 21, 1 ], "matrix": "L0.mlp.W_out" } ], "l:71": [ { "axis0": [ 124, 2 ], "axis1": [ 320, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 33, 1 ], "axis1": [ 320, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 126, 1 ], "axis1": [ 320, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 20, 1 ], "axis1": [ 320, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 18, 1 ], "matrix": "L4.attn.W_O" } ], "l:73": [ { "axis0": [ 0, 1 ], "axis1": [ 124, 1 ], "matrix": "L3.mlp.W_out" } ], "l:74": [ { "axis0": [ 0, 1 ], "axis1": [ 125, 1 ], "matrix": "L3.mlp.W_out" } ], "l:76": [ { "axis0": [ 0, 1 ], "axis1": [ 126, 1 ], "matrix": "L3.mlp.W_out" } ], "l:77": [ { "axis0": [ 0, 1 ], "axis1": [ 201, 62 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 201, 62 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 260, 4 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 272, 13 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 352, 1 ], "axis1": [ 201, 62 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 201, 62 ], "axis1": [ 260, 4 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 201, 62 ], "axis1": [ 272, 13 ], "matrix": "L17.mlp.W_out" } ], "l:78": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 67, 1 ], "axis1": [ 0, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 352, 1 ], "matrix": "L17.attn.W_O" } ], "l:79": [ { "axis0": [ 0, 1 ], "axis1": [ 457, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 271, 2 ], "axis1": [ 457, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 271, 2 ], "axis1": [ 458, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 457, 1 ], "axis1": [ 352, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 458, 1 ], "axis1": [ 352, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" } ], "l:80": [ { "axis0": [ 0, 1 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 26, 3 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 55, 4 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 61, 4 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 68, 8 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1301, 100 ], "axis1": [ 271, 2 ], "matrix": "L11.mlp.W_out" } ], "l:82": [ { "axis0": [ 88, 2 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 90, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 19, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 51, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 71, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 75, 13 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 329, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 52, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 55, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 61, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 68, 8 ], "matrix": "L11.attn.W_O" } ], "l:84": [ { "axis0": [ 0, 1 ], "axis1": [ 88, 1 ], "matrix": "L0.mlp.W_out" } ], "l:85": [ { "axis0": [ 0, 1 ], "axis1": [ 89, 1 ], "matrix": "L0.mlp.W_out" } ], "l:87": [ { "axis0": [ 0, 1 ], "axis1": [ 90, 1 ], "matrix": "L0.mlp.W_out" } ], "l:88": [ { "axis0": [ 35, 1 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 178, 3 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 113, 4 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 19, 2 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 75, 13 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 111, 1 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 329, 1 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 19, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 51, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 71, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 75, 13 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 329, 1 ], "matrix": "L8.attn.W_O" } ], "l:89": [ { "axis0": [ 0, 1 ], "axis1": [ 51, 9 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 51, 9 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 178, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 46, 1 ], "axis1": [ 51, 9 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 51, 9 ], "axis1": [ 35, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 51, 9 ], "axis1": [ 178, 3 ], "matrix": "L4.mlp.W_out" } ], "l:9": [ { "axis0": [ 0, 1 ], "axis1": [ 109, 1 ], "matrix": "L0.mlp.W_out" } ], "l:90": [ { "axis0": [ 46, 2 ], "axis1": [ 128, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 33, 1 ], "axis1": [ 128, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 48, 1 ], "axis1": [ 128, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 24, 1 ], "axis1": [ 128, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 46, 1 ], "matrix": "L4.attn.W_O" } ], "l:92": [ { "axis0": [ 0, 1 ], "axis1": [ 46, 1 ], "matrix": "L0.mlp.W_out" } ], "l:93": [ { "axis0": [ 0, 1 ], "axis1": [ 47, 1 ], "matrix": "L0.mlp.W_out" } ], "l:95": [ { "axis0": [ 0, 1 ], "axis1": [ 48, 1 ], "matrix": "L0.mlp.W_out" } ], "l:96": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 27, 1 ], "axis1": [ 0, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 24, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 24, 1 ], "matrix": "L3.mlp.W_out" } ], "l:97": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 32 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 32 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 27, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 3, 32 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 3, 32 ], "axis1": [ 27, 1 ], "matrix": "L2.mlp.W_out" } ], "l:98": [ { "axis0": [ 33, 1 ], "axis1": [ 0, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 28, 1 ], "axis1": [ 0, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 117, 1 ], "axis1": [ 0, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 24, 1 ], "matrix": "L2.attn.W_O" } ], "l:99": [ { "axis0": [ 0, 1 ], "axis1": [ 33, 1 ], "matrix": "L0.mlp.W_out" } ], "physical:cancel": [ { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 27, 3 ], "axis1": [ 192, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 107, 3 ], "axis1": [ 192, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 27, 3 ], "matrix": "L1.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 107, 3 ], "matrix": "L1.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 27, 2 ], "axis1": [ 128, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 33, 1 ], "axis1": [ 128, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 117, 3 ], "axis1": [ 128, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 27, 2 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 33, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 117, 3 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 24, 2 ], "axis1": [ 128, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 24, 2 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 18, 3 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 22, 1 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 30, 3 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 35, 3 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 46, 3 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 117, 1 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 120, 7 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 18, 3 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 22, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 30, 3 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 35, 3 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 46, 3 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 117, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 120, 7 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 34, 1 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 36, 6 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 46, 1 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 107, 1 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 110, 1 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 181, 54 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 34, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 36, 6 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 46, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 107, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 110, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 181, 54 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 235, 14 ], "axis1": [ 320, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 296, 1 ], "axis1": [ 320, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 235, 14 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 296, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 33, 1 ], "axis1": [ 64, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 33, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 1, 17 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 49, 3 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 1, 17 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 49, 3 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 19, 2 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 26, 3 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 30, 3 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 35, 1 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 51, 1 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 75, 13 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 111, 1 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 113, 4 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 125, 2 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 178, 3 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 329, 1 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 19, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 26, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 30, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 35, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 51, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 75, 13 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 111, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 113, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 125, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 178, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 329, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 23, 1 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 57, 2 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 61, 4 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 66, 1 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 68, 2 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 117, 1 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 121, 4 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 155, 17 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 330, 1 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 23, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 57, 2 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 61, 4 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 66, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 68, 2 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 117, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 121, 4 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 155, 17 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 330, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 19, 5 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 26, 3 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 30, 6 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 38, 2 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 49, 10 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 61, 4 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 66, 1 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 68, 2 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 71, 31 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 19, 5 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 26, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 30, 6 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 38, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 49, 10 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 61, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 66, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 68, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 71, 31 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 102, 4 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 111, 1 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 113, 5 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 120, 7 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 153, 19 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 178, 3 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 249, 9 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 261, 6 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 327, 1 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 329, 2 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 102, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 111, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 113, 5 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 120, 7 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 153, 19 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 178, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 249, 9 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 261, 6 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 327, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 329, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 261, 2 ], "axis1": [ 128, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 268, 1 ], "axis1": [ 128, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 261, 2 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 268, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 106, 1 ], "axis1": [ 64, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 267, 1 ], "axis1": [ 64, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 106, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 267, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 50, 1 ], "axis1": [ 64, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 349, 1 ], "axis1": [ 64, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 50, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 349, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 323, 1 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 350, 1 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 323, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 350, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 22, 1 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 47, 1 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 272, 1 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 351, 1 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 353, 16 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 22, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 47, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 272, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 351, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 353, 16 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_K" }, { "axis0": [ 42, 3 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 65, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 67, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 207, 5 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 213, 4 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 218, 4 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 227, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 285, 5 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 324, 3 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 346, 3 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 351, 34 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 42, 3 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 65, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 67, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 207, 5 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 213, 4 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 218, 4 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 227, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 285, 5 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 324, 3 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 346, 3 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 351, 34 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_K" }, { "axis0": [ 385, 1 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 420, 5 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 385, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 420, 5 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_K" }, { "axis0": [ 36, 2 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 42, 3 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 172, 6 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 181, 26 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 213, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 215, 2 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 218, 4 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 227, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 241, 4 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 334, 2 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 513, 13 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 36, 2 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 42, 3 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 172, 6 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 181, 26 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 213, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 215, 2 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 218, 4 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 227, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 241, 4 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 334, 2 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 513, 13 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L19.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L19.attn.W_K" }, { "axis0": [ 526, 4 ], "axis1": [ 192, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 526, 4 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L22.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L22.attn.W_K" }, { "axis0": [ 54, 3 ], "axis1": [ 64, 64 ], "matrix": "L22.attn.W_V" }, { "axis0": [ 168, 1 ], "axis1": [ 64, 64 ], "matrix": "L22.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 54, 3 ], "matrix": "L22.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 168, 1 ], "matrix": "L22.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L23.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L23.attn.W_K" }, { "axis0": [ 205, 1 ], "axis1": [ 0, 64 ], "matrix": "L23.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 205, 1 ], "matrix": "L23.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L24.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L24.attn.W_K" }, { "axis0": [ 50, 1 ], "axis1": [ 0, 64 ], "matrix": "L24.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 50, 1 ], "matrix": "L24.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_K" }, { "axis0": [ 30, 4 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 37, 3 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 42, 1 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 53, 4 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 76, 2 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 141, 26 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 168, 4 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 185, 12 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 211, 1 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 213, 3 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 327, 1 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 333, 2 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 341, 1 ], "axis1": [ 192, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 30, 4 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 37, 3 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 42, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 53, 4 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 76, 2 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 141, 26 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 168, 4 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 185, 12 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 211, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 213, 3 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 327, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 333, 2 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 341, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L26.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L26.attn.W_K" }, { "axis0": [ 1895, 5 ], "axis1": [ 256, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 1895, 5 ], "matrix": "L26.attn.W_O" } ], "physical:cancel_bypass": [ { "axis0": [ 0, 1 ], "axis1": [ 19, 8 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 38, 4 ], "axis1": [ 19, 4 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 38, 4 ], "axis1": [ 23, 4 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 19, 4 ], "axis1": [ 38, 4 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 23, 4 ], "axis1": [ 38, 4 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 27, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 24, 1 ], "axis1": [ 27, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 24, 1 ], "axis1": [ 28, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 24, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 28, 1 ], "axis1": [ 24, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 29, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 25, 1 ], "axis1": [ 29, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 25, 1 ], "axis1": [ 30, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 29, 1 ], "axis1": [ 25, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 30, 1 ], "axis1": [ 25, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 31, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 26, 1 ], "axis1": [ 31, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 26, 1 ], "axis1": [ 32, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 31, 1 ], "axis1": [ 26, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 32, 1 ], "axis1": [ 26, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 33, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 22, 1 ], "axis1": [ 33, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 22, 1 ], "axis1": [ 34, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 22, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 34, 1 ], "axis1": [ 22, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 67, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 108, 1 ], "axis1": [ 67, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 108, 1 ], "axis1": [ 68, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 108, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 68, 1 ], "axis1": [ 108, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 107, 1 ], "axis1": [ 69, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 107, 1 ], "axis1": [ 70, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 107, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 70, 1 ], "axis1": [ 107, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 71, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 110, 1 ], "axis1": [ 71, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 72, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 71, 1 ], "axis1": [ 110, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 72, 1 ], "axis1": [ 110, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 17, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 26, 1 ], "axis1": [ 17, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 26, 1 ], "axis1": [ 18, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 17, 1 ], "axis1": [ 26, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 18, 1 ], "axis1": [ 26, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 19, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 111, 1 ], "axis1": [ 19, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 20, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 19, 1 ], "axis1": [ 111, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 20, 1 ], "axis1": [ 111, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 21, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 21, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 22, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 21, 1 ], "axis1": [ 27, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 22, 1 ], "axis1": [ 27, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 23, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 28, 1 ], "axis1": [ 23, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 24, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 23, 1 ], "axis1": [ 28, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 28, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 190, 34 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 127, 17 ], "axis1": [ 190, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 127, 17 ], "axis1": [ 207, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 190, 17 ], "axis1": [ 127, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 207, 17 ], "axis1": [ 127, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 224, 34 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 144, 17 ], "axis1": [ 224, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 144, 17 ], "axis1": [ 241, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 224, 17 ], "axis1": [ 144, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 241, 17 ], "axis1": [ 144, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 258, 34 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 161, 17 ], "axis1": [ 258, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 161, 17 ], "axis1": [ 275, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 258, 17 ], "axis1": [ 161, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 275, 17 ], "axis1": [ 161, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 292, 8 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 42, 4 ], "axis1": [ 292, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 42, 4 ], "axis1": [ 296, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 292, 4 ], "axis1": [ 42, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 296, 4 ], "axis1": [ 42, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 300, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 300, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 301, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 300, 1 ], "axis1": [ 33, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 301, 1 ], "axis1": [ 33, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 302, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 112, 1 ], "axis1": [ 302, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 112, 1 ], "axis1": [ 303, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 112, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 303, 1 ], "axis1": [ 112, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 304, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 24, 1 ], "axis1": [ 304, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 24, 1 ], "axis1": [ 305, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 304, 1 ], "axis1": [ 24, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 305, 1 ], "axis1": [ 24, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 306, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 108, 1 ], "axis1": [ 306, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 108, 1 ], "axis1": [ 307, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 306, 1 ], "axis1": [ 108, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 307, 1 ], "axis1": [ 108, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 308, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 25, 1 ], "axis1": [ 308, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 25, 1 ], "axis1": [ 309, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 308, 1 ], "axis1": [ 25, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 309, 1 ], "axis1": [ 25, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 164, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 71, 17 ], "axis1": [ 164, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 71, 17 ], "axis1": [ 181, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 164, 17 ], "axis1": [ 71, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 181, 17 ], "axis1": [ 71, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 198, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 297, 17 ], "axis1": [ 198, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 297, 17 ], "axis1": [ 215, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 198, 17 ], "axis1": [ 297, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 215, 17 ], "axis1": [ 297, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 232, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 314, 17 ], "axis1": [ 232, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 314, 17 ], "axis1": [ 249, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 232, 17 ], "axis1": [ 314, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 249, 17 ], "axis1": [ 314, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 266, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 249, 17 ], "axis1": [ 266, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 249, 17 ], "axis1": [ 283, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 266, 17 ], "axis1": [ 249, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 283, 17 ], "axis1": [ 249, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 300, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 266, 17 ], "axis1": [ 300, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 266, 17 ], "axis1": [ 317, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 300, 17 ], "axis1": [ 266, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 317, 17 ], "axis1": [ 266, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 334, 6 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 22, 1 ], "axis1": [ 334, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 334, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 117, 1 ], "axis1": [ 334, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 22, 1 ], "axis1": [ 337, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 337, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 117, 1 ], "axis1": [ 337, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 22, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 335, 1 ], "axis1": [ 111, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 336, 1 ], "axis1": [ 117, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 337, 1 ], "axis1": [ 22, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 338, 1 ], "axis1": [ 111, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 339, 1 ], "axis1": [ 117, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 340, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 23, 1 ], "axis1": [ 340, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 23, 1 ], "axis1": [ 341, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 340, 1 ], "axis1": [ 23, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 341, 1 ], "axis1": [ 23, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 342, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 20, 1 ], "axis1": [ 342, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 20, 1 ], "axis1": [ 343, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 342, 1 ], "axis1": [ 20, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 343, 1 ], "axis1": [ 20, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 344, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 19, 1 ], "axis1": [ 344, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 19, 1 ], "axis1": [ 345, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 344, 1 ], "axis1": [ 19, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 345, 1 ], "axis1": [ 19, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 346, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 283, 1 ], "axis1": [ 346, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 283, 1 ], "axis1": [ 347, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 346, 1 ], "axis1": [ 283, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 347, 1 ], "axis1": [ 283, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 348, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 284, 1 ], "axis1": [ 348, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 284, 1 ], "axis1": [ 349, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 348, 1 ], "axis1": [ 284, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 349, 1 ], "axis1": [ 284, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 350, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 285, 1 ], "axis1": [ 350, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 285, 1 ], "axis1": [ 351, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 350, 1 ], "axis1": [ 285, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 351, 1 ], "axis1": [ 285, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 352, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 286, 1 ], "axis1": [ 352, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 286, 1 ], "axis1": [ 353, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 352, 1 ], "axis1": [ 286, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 353, 1 ], "axis1": [ 286, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 354, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 287, 1 ], "axis1": [ 354, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 287, 1 ], "axis1": [ 355, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 354, 1 ], "axis1": [ 287, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 355, 1 ], "axis1": [ 287, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 356, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 288, 1 ], "axis1": [ 356, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 288, 1 ], "axis1": [ 357, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 356, 1 ], "axis1": [ 288, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 357, 1 ], "axis1": [ 288, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 358, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 289, 1 ], "axis1": [ 358, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 289, 1 ], "axis1": [ 359, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 358, 1 ], "axis1": [ 289, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 359, 1 ], "axis1": [ 289, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 360, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 290, 1 ], "axis1": [ 360, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 290, 1 ], "axis1": [ 361, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 360, 1 ], "axis1": [ 290, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 361, 1 ], "axis1": [ 290, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 362, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 291, 1 ], "axis1": [ 362, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 291, 1 ], "axis1": [ 363, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 362, 1 ], "axis1": [ 291, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 363, 1 ], "axis1": [ 291, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 364, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 292, 1 ], "axis1": [ 364, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 292, 1 ], "axis1": [ 365, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 364, 1 ], "axis1": [ 292, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 365, 1 ], "axis1": [ 292, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 40, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 53, 17 ], "axis1": [ 40, 17 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 53, 17 ], "axis1": [ 57, 17 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 40, 17 ], "axis1": [ 53, 17 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 57, 17 ], "axis1": [ 53, 17 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 74, 6 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 34, 1 ], "axis1": [ 74, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 74, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 34, 1 ], "axis1": [ 77, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 77, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 74, 1 ], "axis1": [ 34, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 75, 2 ], "axis1": [ 38, 2 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 77, 1 ], "axis1": [ 34, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 78, 2 ], "axis1": [ 38, 2 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 80, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 154, 1 ], "axis1": [ 80, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 154, 1 ], "axis1": [ 81, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 80, 1 ], "axis1": [ 154, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 81, 1 ], "axis1": [ 154, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 82, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 153, 1 ], "axis1": [ 82, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 153, 1 ], "axis1": [ 83, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 82, 1 ], "axis1": [ 153, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 83, 1 ], "axis1": [ 153, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 52, 2 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 71, 1 ], "axis1": [ 52, 1 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 71, 1 ], "axis1": [ 53, 1 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 71, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 53, 1 ], "axis1": [ 71, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 54, 2 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 22, 1 ], "axis1": [ 54, 1 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 22, 1 ], "axis1": [ 55, 1 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 54, 1 ], "axis1": [ 22, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 55, 1 ], "axis1": [ 22, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 4, 6 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 72, 3 ], "axis1": [ 4, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 72, 3 ], "axis1": [ 7, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 4, 3 ], "axis1": [ 72, 3 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 7, 3 ], "axis1": [ 72, 3 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 10, 6 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 54, 3 ], "axis1": [ 10, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 54, 3 ], "axis1": [ 13, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 10, 3 ], "axis1": [ 54, 3 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 13, 3 ], "axis1": [ 54, 3 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 16, 2 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 120, 1 ], "axis1": [ 16, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 120, 1 ], "axis1": [ 17, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 16, 1 ], "axis1": [ 120, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 17, 1 ], "axis1": [ 120, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 18, 2 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 52, 1 ], "axis1": [ 18, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 19, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 52, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 19, 1 ], "axis1": [ 52, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 20, 2 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 53, 1 ], "axis1": [ 20, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 53, 1 ], "axis1": [ 21, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 20, 1 ], "axis1": [ 53, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 21, 1 ], "axis1": [ 53, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 6 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 255, 3 ], "axis1": [ 1, 3 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 255, 3 ], "axis1": [ 4, 3 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1, 3 ], "axis1": [ 255, 3 ], "diagonal": true, "matrix": "L9.mlp.W_out" }, { "axis0": [ 4, 3 ], "axis1": [ 255, 3 ], "diagonal": true, "matrix": "L9.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 6 ], "matrix": "L10.mlp.W_up" }, { "axis0": [ 258, 3 ], "axis1": [ 1, 3 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 258, 3 ], "axis1": [ 4, 3 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 1, 3 ], "axis1": [ 258, 3 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 4, 3 ], "axis1": [ 258, 3 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L10.mlp.W_up" }, { "axis0": [ 70, 1 ], "axis1": [ 7, 1 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 70, 1 ], "axis1": [ 8, 1 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 7, 1 ], "axis1": [ 70, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 8, 1 ], "axis1": [ 70, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1601, 4 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 267, 2 ], "axis1": [ 1601, 2 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 267, 2 ], "axis1": [ 1603, 2 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1601, 2 ], "axis1": [ 267, 2 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1603, 2 ], "axis1": [ 267, 2 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 459, 4 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 269, 2 ], "axis1": [ 459, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 269, 2 ], "axis1": [ 461, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 459, 2 ], "axis1": [ 269, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 461, 2 ], "axis1": [ 269, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 463, 4 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 271, 2 ], "axis1": [ 463, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 271, 2 ], "axis1": [ 465, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 463, 2 ], "axis1": [ 271, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 465, 2 ], "axis1": [ 271, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 467, 4 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 265, 2 ], "axis1": [ 467, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 265, 2 ], "axis1": [ 469, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 467, 2 ], "axis1": [ 265, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 469, 2 ], "axis1": [ 265, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 471, 4 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 259, 2 ], "axis1": [ 471, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 259, 2 ], "axis1": [ 473, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 471, 2 ], "axis1": [ 259, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 473, 2 ], "axis1": [ 259, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 475, 4 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 53, 2 ], "axis1": [ 475, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 53, 2 ], "axis1": [ 477, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 475, 2 ], "axis1": [ 53, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 477, 2 ], "axis1": [ 53, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 479, 4 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 253, 2 ], "axis1": [ 479, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 253, 2 ], "axis1": [ 481, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 479, 2 ], "axis1": [ 253, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 481, 2 ], "axis1": [ 253, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 483, 4 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 263, 2 ], "axis1": [ 483, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 263, 2 ], "axis1": [ 485, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 483, 2 ], "axis1": [ 263, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 485, 2 ], "axis1": [ 263, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 487, 4 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 256, 2 ], "axis1": [ 487, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 256, 2 ], "axis1": [ 489, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 487, 2 ], "axis1": [ 256, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 489, 2 ], "axis1": [ 256, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 491, 4 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 51, 1 ], "axis1": [ 491, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 258, 1 ], "axis1": [ 491, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 51, 1 ], "axis1": [ 493, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 258, 1 ], "axis1": [ 493, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 491, 1 ], "axis1": [ 51, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 492, 1 ], "axis1": [ 258, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 493, 1 ], "axis1": [ 51, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 494, 1 ], "axis1": [ 258, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 495, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 250, 1 ], "axis1": [ 495, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 250, 1 ], "axis1": [ 496, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 495, 1 ], "axis1": [ 250, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 496, 1 ], "axis1": [ 250, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 497, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 251, 1 ], "axis1": [ 497, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 251, 1 ], "axis1": [ 498, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 497, 1 ], "axis1": [ 251, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 498, 1 ], "axis1": [ 251, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 499, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 50, 1 ], "axis1": [ 499, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 50, 1 ], "axis1": [ 500, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 499, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 500, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 559, 34 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 300, 17 ], "axis1": [ 559, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 300, 17 ], "axis1": [ 576, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 559, 17 ], "axis1": [ 300, 17 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 576, 17 ], "axis1": [ 300, 17 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 593, 34 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 65, 1 ], "axis1": [ 593, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 593, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 317, 2 ], "axis1": [ 593, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 320, 7 ], "axis1": [ 593, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 593, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 344, 5 ], "axis1": [ 593, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 610, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 610, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 317, 2 ], "axis1": [ 610, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 320, 7 ], "axis1": [ 610, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 610, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 344, 5 ], "axis1": [ 610, 17 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 593, 1 ], "axis1": [ 65, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 594, 1 ], "axis1": [ 67, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 595, 2 ], "axis1": [ 317, 2 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 597, 7 ], "axis1": [ 320, 7 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 604, 1 ], "axis1": [ 328, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 605, 5 ], "axis1": [ 344, 5 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 610, 1 ], "axis1": [ 65, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 611, 1 ], "axis1": [ 67, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 612, 2 ], "axis1": [ 317, 2 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 614, 7 ], "axis1": [ 320, 7 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 621, 1 ], "axis1": [ 328, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 622, 5 ], "axis1": [ 344, 5 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 627, 4 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 298, 2 ], "axis1": [ 627, 2 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 298, 2 ], "axis1": [ 629, 2 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 627, 2 ], "axis1": [ 298, 2 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 629, 2 ], "axis1": [ 298, 2 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 631, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 290, 1 ], "axis1": [ 631, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 290, 1 ], "axis1": [ 632, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 631, 1 ], "axis1": [ 290, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 632, 1 ], "axis1": [ 290, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 633, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 291, 1 ], "axis1": [ 633, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 291, 1 ], "axis1": [ 634, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 633, 1 ], "axis1": [ 291, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 634, 1 ], "axis1": [ 291, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 635, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 292, 1 ], "axis1": [ 635, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 292, 1 ], "axis1": [ 636, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 635, 1 ], "axis1": [ 292, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 636, 1 ], "axis1": [ 292, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 637, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 60, 1 ], "axis1": [ 637, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 60, 1 ], "axis1": [ 638, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 637, 1 ], "axis1": [ 60, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 638, 1 ], "axis1": [ 60, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 639, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 59, 1 ], "axis1": [ 639, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 59, 1 ], "axis1": [ 640, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 639, 1 ], "axis1": [ 59, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 640, 1 ], "axis1": [ 59, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 641, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 297, 1 ], "axis1": [ 641, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 297, 1 ], "axis1": [ 642, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 641, 1 ], "axis1": [ 297, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 642, 1 ], "axis1": [ 297, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 642, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 52, 1 ], "axis1": [ 642, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 55, 4 ], "axis1": [ 642, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 61, 4 ], "axis1": [ 642, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 68, 8 ], "axis1": [ 642, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 659, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 55, 4 ], "axis1": [ 659, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 61, 4 ], "axis1": [ 659, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 68, 8 ], "axis1": [ 659, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 642, 1 ], "axis1": [ 52, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 643, 4 ], "axis1": [ 55, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 647, 4 ], "axis1": [ 61, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 651, 8 ], "axis1": [ 68, 8 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 659, 1 ], "axis1": [ 52, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 660, 4 ], "axis1": [ 55, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 664, 4 ], "axis1": [ 61, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 668, 8 ], "axis1": [ 68, 8 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 676, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 76, 13 ], "axis1": [ 676, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 90, 3 ], "axis1": [ 676, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 676, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 76, 13 ], "axis1": [ 693, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 90, 3 ], "axis1": [ 693, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 693, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 676, 13 ], "axis1": [ 76, 13 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 689, 3 ], "axis1": [ 90, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 692, 1 ], "axis1": [ 329, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 693, 13 ], "axis1": [ 76, 13 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 706, 3 ], "axis1": [ 90, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 709, 1 ], "axis1": [ 329, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 710, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 19, 5 ], "axis1": [ 710, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 93, 3 ], "axis1": [ 710, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 97, 4 ], "axis1": [ 710, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 710, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 710, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 19, 5 ], "axis1": [ 727, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 93, 3 ], "axis1": [ 727, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 97, 4 ], "axis1": [ 727, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 727, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 727, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 710, 5 ], "axis1": [ 19, 5 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 715, 3 ], "axis1": [ 93, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 718, 4 ], "axis1": [ 97, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 722, 4 ], "axis1": [ 102, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 726, 1 ], "axis1": [ 111, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 727, 5 ], "axis1": [ 19, 5 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 732, 3 ], "axis1": [ 93, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 735, 4 ], "axis1": [ 97, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 739, 4 ], "axis1": [ 102, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 743, 1 ], "axis1": [ 111, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 744, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 26, 3 ], "axis1": [ 744, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 744, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 744, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 744, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 744, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 26, 3 ], "axis1": [ 761, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 761, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 761, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 761, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 761, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 744, 3 ], "axis1": [ 26, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 747, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 748, 5 ], "axis1": [ 113, 5 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 753, 7 ], "axis1": [ 120, 7 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 760, 1 ], "axis1": [ 330, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 761, 3 ], "axis1": [ 26, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 764, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 765, 5 ], "axis1": [ 113, 5 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 770, 7 ], "axis1": [ 120, 7 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 777, 1 ], "axis1": [ 330, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 778, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 30, 3 ], "axis1": [ 778, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 153, 14 ], "axis1": [ 778, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 30, 3 ], "axis1": [ 795, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 153, 14 ], "axis1": [ 795, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 778, 3 ], "axis1": [ 30, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 781, 14 ], "axis1": [ 153, 14 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 795, 3 ], "axis1": [ 30, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 798, 14 ], "axis1": [ 153, 14 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 812, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 33, 3 ], "axis1": [ 812, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 812, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 812, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 812, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 812, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 249, 1 ], "axis1": [ 812, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 252, 1 ], "axis1": [ 812, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 255, 1 ], "axis1": [ 812, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 812, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 33, 3 ], "axis1": [ 829, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 829, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 829, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 829, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 829, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 249, 1 ], "axis1": [ 829, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 252, 1 ], "axis1": [ 829, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 255, 1 ], "axis1": [ 829, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 829, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 812, 3 ], "axis1": [ 33, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 815, 2 ], "axis1": [ 38, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 817, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 818, 4 ], "axis1": [ 168, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 822, 3 ], "axis1": [ 178, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 825, 1 ], "axis1": [ 249, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 826, 1 ], "axis1": [ 252, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 827, 1 ], "axis1": [ 255, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 828, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 829, 3 ], "axis1": [ 33, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 832, 2 ], "axis1": [ 38, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 834, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 835, 4 ], "axis1": [ 168, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 839, 3 ], "axis1": [ 178, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 842, 1 ], "axis1": [ 249, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 843, 1 ], "axis1": [ 252, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 844, 1 ], "axis1": [ 255, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 845, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 846, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 403, 17 ], "axis1": [ 846, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 403, 17 ], "axis1": [ 863, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 846, 17 ], "axis1": [ 403, 17 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 863, 17 ], "axis1": [ 403, 17 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 880, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 51, 1 ], "axis1": [ 880, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 53, 2 ], "axis1": [ 880, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 256, 11 ], "axis1": [ 880, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 269, 3 ], "axis1": [ 880, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 51, 1 ], "axis1": [ 897, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 53, 2 ], "axis1": [ 897, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 256, 11 ], "axis1": [ 897, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 269, 3 ], "axis1": [ 897, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 880, 1 ], "axis1": [ 51, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 881, 2 ], "axis1": [ 53, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 883, 11 ], "axis1": [ 256, 11 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 894, 3 ], "axis1": [ 269, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 897, 1 ], "axis1": [ 51, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 898, 2 ], "axis1": [ 53, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 900, 11 ], "axis1": [ 256, 11 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 911, 3 ], "axis1": [ 269, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 914, 4 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 253, 2 ], "axis1": [ 914, 2 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 253, 2 ], "axis1": [ 916, 2 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 914, 2 ], "axis1": [ 253, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 916, 2 ], "axis1": [ 253, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 918, 2 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 250, 1 ], "axis1": [ 918, 1 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 250, 1 ], "axis1": [ 919, 1 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 918, 1 ], "axis1": [ 250, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 919, 1 ], "axis1": [ 250, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 920, 2 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 251, 1 ], "axis1": [ 920, 1 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 251, 1 ], "axis1": [ 921, 1 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 920, 1 ], "axis1": [ 251, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 921, 1 ], "axis1": [ 251, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 655, 34 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 479, 17 ], "axis1": [ 655, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 479, 17 ], "axis1": [ 672, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 655, 17 ], "axis1": [ 479, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 672, 17 ], "axis1": [ 479, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 689, 34 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 59, 2 ], "axis1": [ 689, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 689, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 689, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 267, 1 ], "axis1": [ 689, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 290, 3 ], "axis1": [ 689, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 297, 9 ], "axis1": [ 689, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 59, 2 ], "axis1": [ 706, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 706, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 706, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 267, 1 ], "axis1": [ 706, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 290, 3 ], "axis1": [ 706, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 297, 9 ], "axis1": [ 706, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 689, 2 ], "axis1": [ 59, 2 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 691, 1 ], "axis1": [ 67, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 692, 1 ], "axis1": [ 106, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 693, 1 ], "axis1": [ 267, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 694, 3 ], "axis1": [ 290, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 697, 9 ], "axis1": [ 297, 9 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 706, 2 ], "axis1": [ 59, 2 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 708, 1 ], "axis1": [ 67, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 709, 1 ], "axis1": [ 106, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 710, 1 ], "axis1": [ 267, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 711, 3 ], "axis1": [ 290, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 714, 9 ], "axis1": [ 297, 9 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 723, 34 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 306, 13 ], "axis1": [ 723, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 723, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 723, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 306, 13 ], "axis1": [ 740, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 740, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 740, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 723, 13 ], "axis1": [ 306, 13 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 736, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 739, 1 ], "axis1": [ 328, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 740, 13 ], "axis1": [ 306, 13 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 753, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 756, 1 ], "axis1": [ 328, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 757, 34 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 496, 17 ], "axis1": [ 757, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 496, 17 ], "axis1": [ 774, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 757, 17 ], "axis1": [ 496, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 774, 17 ], "axis1": [ 496, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 791, 4 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 531, 2 ], "axis1": [ 791, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 531, 2 ], "axis1": [ 793, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 791, 2 ], "axis1": [ 531, 2 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 793, 2 ], "axis1": [ 531, 2 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 795, 2 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 530, 1 ], "axis1": [ 795, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 530, 1 ], "axis1": [ 796, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 795, 1 ], "axis1": [ 530, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 796, 1 ], "axis1": [ 530, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 615, 34 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 273, 17 ], "axis1": [ 615, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 273, 17 ], "axis1": [ 632, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 615, 17 ], "axis1": [ 273, 17 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 632, 17 ], "axis1": [ 273, 17 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 649, 34 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 65, 1 ], "axis1": [ 649, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 649, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 344, 5 ], "axis1": [ 649, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 420, 8 ], "axis1": [ 649, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 666, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 666, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 344, 5 ], "axis1": [ 666, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 420, 8 ], "axis1": [ 666, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 649, 1 ], "axis1": [ 65, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 650, 3 ], "axis1": [ 324, 3 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 653, 5 ], "axis1": [ 344, 5 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 658, 8 ], "axis1": [ 420, 8 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 666, 1 ], "axis1": [ 65, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 667, 3 ], "axis1": [ 324, 3 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 670, 5 ], "axis1": [ 344, 5 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 675, 8 ], "axis1": [ 420, 8 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 683, 34 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 428, 17 ], "axis1": [ 683, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 428, 17 ], "axis1": [ 700, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 683, 17 ], "axis1": [ 428, 17 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 700, 17 ], "axis1": [ 428, 17 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 717, 34 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 49, 1 ], "axis1": [ 717, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 51, 2 ], "axis1": [ 717, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 249, 2 ], "axis1": [ 717, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 252, 12 ], "axis1": [ 717, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 734, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 51, 2 ], "axis1": [ 734, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 249, 2 ], "axis1": [ 734, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 252, 12 ], "axis1": [ 734, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 717, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 718, 2 ], "axis1": [ 51, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 720, 2 ], "axis1": [ 249, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 722, 12 ], "axis1": [ 252, 12 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 734, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 735, 2 ], "axis1": [ 51, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 737, 2 ], "axis1": [ 249, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 739, 12 ], "axis1": [ 252, 12 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 751, 34 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 23, 1 ], "axis1": [ 751, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 26, 2 ], "axis1": [ 751, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 751, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 751, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 113, 3 ], "axis1": [ 751, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 117, 1 ], "axis1": [ 751, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 751, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 751, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 23, 1 ], "axis1": [ 768, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 26, 2 ], "axis1": [ 768, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 768, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 768, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 113, 3 ], "axis1": [ 768, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 117, 1 ], "axis1": [ 768, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 768, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 768, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 751, 1 ], "axis1": [ 23, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 752, 2 ], "axis1": [ 26, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 754, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 755, 1 ], "axis1": [ 111, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 756, 3 ], "axis1": [ 113, 3 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 759, 1 ], "axis1": [ 117, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 760, 7 ], "axis1": [ 120, 7 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 767, 1 ], "axis1": [ 330, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 768, 1 ], "axis1": [ 23, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 769, 2 ], "axis1": [ 26, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 771, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 772, 1 ], "axis1": [ 111, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 773, 3 ], "axis1": [ 113, 3 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 776, 1 ], "axis1": [ 117, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 777, 7 ], "axis1": [ 120, 7 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 784, 1 ], "axis1": [ 330, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 785, 4 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 21, 1 ], "axis1": [ 785, 2 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 116, 1 ], "axis1": [ 785, 2 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 21, 1 ], "axis1": [ 787, 2 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 116, 1 ], "axis1": [ 787, 2 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 785, 1 ], "axis1": [ 21, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 786, 1 ], "axis1": [ 116, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 787, 1 ], "axis1": [ 21, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 788, 1 ], "axis1": [ 116, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 332, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 445, 17 ], "axis1": [ 332, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 445, 17 ], "axis1": [ 349, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 332, 17 ], "axis1": [ 445, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 349, 17 ], "axis1": [ 445, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 366, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 462, 17 ], "axis1": [ 366, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 462, 17 ], "axis1": [ 383, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 366, 17 ], "axis1": [ 462, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 383, 17 ], "axis1": [ 462, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 400, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 61, 2 ], "axis1": [ 400, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 298, 9 ], "axis1": [ 400, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 400, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 313, 2 ], "axis1": [ 400, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 61, 2 ], "axis1": [ 417, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 298, 9 ], "axis1": [ 417, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 417, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 313, 2 ], "axis1": [ 417, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 400, 2 ], "axis1": [ 61, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 402, 9 ], "axis1": [ 298, 9 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 411, 4 ], "axis1": [ 308, 4 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 415, 2 ], "axis1": [ 313, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 417, 2 ], "axis1": [ 61, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 419, 9 ], "axis1": [ 298, 9 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 428, 4 ], "axis1": [ 308, 4 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 432, 2 ], "axis1": [ 313, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 434, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 28, 1 ], "axis1": [ 434, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 30, 3 ], "axis1": [ 434, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 153, 13 ], "axis1": [ 434, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 451, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 30, 3 ], "axis1": [ 451, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 153, 13 ], "axis1": [ 451, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 434, 1 ], "axis1": [ 28, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 435, 3 ], "axis1": [ 30, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 438, 13 ], "axis1": [ 153, 13 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 451, 1 ], "axis1": [ 28, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 452, 3 ], "axis1": [ 30, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 455, 13 ], "axis1": [ 153, 13 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 468, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 241, 8 ], "axis1": [ 468, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 296, 1 ], "axis1": [ 468, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 331, 8 ], "axis1": [ 468, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 241, 8 ], "axis1": [ 485, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 296, 1 ], "axis1": [ 485, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 331, 8 ], "axis1": [ 485, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 468, 8 ], "axis1": [ 241, 8 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 476, 1 ], "axis1": [ 296, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 477, 8 ], "axis1": [ 331, 8 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 485, 8 ], "axis1": [ 241, 8 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 493, 1 ], "axis1": [ 296, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 494, 8 ], "axis1": [ 331, 8 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 502, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 63, 2 ], "axis1": [ 502, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 70, 6 ], "axis1": [ 502, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 315, 2 ], "axis1": [ 502, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 318, 1 ], "axis1": [ 502, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 502, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 502, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 349, 2 ], "axis1": [ 502, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 63, 2 ], "axis1": [ 519, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 70, 6 ], "axis1": [ 519, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 315, 2 ], "axis1": [ 519, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 318, 1 ], "axis1": [ 519, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 519, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 519, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 349, 2 ], "axis1": [ 519, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 502, 2 ], "axis1": [ 63, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 504, 6 ], "axis1": [ 70, 6 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 510, 2 ], "axis1": [ 315, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 512, 1 ], "axis1": [ 318, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 513, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 516, 1 ], "axis1": [ 328, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 517, 2 ], "axis1": [ 349, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 519, 2 ], "axis1": [ 63, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 521, 6 ], "axis1": [ 70, 6 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 527, 2 ], "axis1": [ 315, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 529, 1 ], "axis1": [ 318, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 530, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 533, 1 ], "axis1": [ 328, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 534, 2 ], "axis1": [ 349, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 536, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 83, 6 ], "axis1": [ 536, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 90, 5 ], "axis1": [ 536, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 536, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 414, 1 ], "axis1": [ 536, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 416, 4 ], "axis1": [ 536, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 83, 6 ], "axis1": [ 553, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 90, 5 ], "axis1": [ 553, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 553, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 414, 1 ], "axis1": [ 553, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 416, 4 ], "axis1": [ 553, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 536, 6 ], "axis1": [ 83, 6 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 542, 5 ], "axis1": [ 90, 5 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 547, 1 ], "axis1": [ 329, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 548, 1 ], "axis1": [ 414, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 549, 4 ], "axis1": [ 416, 4 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 553, 6 ], "axis1": [ 83, 6 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 559, 5 ], "axis1": [ 90, 5 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 564, 1 ], "axis1": [ 329, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 565, 1 ], "axis1": [ 414, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 566, 4 ], "axis1": [ 416, 4 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 570, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 19, 1 ], "axis1": [ 570, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 570, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 570, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 479, 10 ], "axis1": [ 570, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 490, 4 ], "axis1": [ 570, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 19, 1 ], "axis1": [ 587, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 587, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 587, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 479, 10 ], "axis1": [ 587, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 490, 4 ], "axis1": [ 587, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 570, 1 ], "axis1": [ 19, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 571, 1 ], "axis1": [ 95, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 572, 1 ], "axis1": [ 97, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 573, 10 ], "axis1": [ 479, 10 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 583, 4 ], "axis1": [ 490, 4 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 587, 1 ], "axis1": [ 19, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 588, 1 ], "axis1": [ 95, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 589, 1 ], "axis1": [ 97, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 590, 10 ], "axis1": [ 479, 10 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 600, 4 ], "axis1": [ 490, 4 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 604, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 386, 17 ], "axis1": [ 604, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 386, 17 ], "axis1": [ 621, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 604, 17 ], "axis1": [ 386, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 621, 17 ], "axis1": [ 386, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 638, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 53, 8 ], "axis1": [ 638, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 265, 2 ], "axis1": [ 638, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 269, 3 ], "axis1": [ 638, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 290, 3 ], "axis1": [ 638, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 297, 1 ], "axis1": [ 638, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 53, 8 ], "axis1": [ 655, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 265, 2 ], "axis1": [ 655, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 269, 3 ], "axis1": [ 655, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 290, 3 ], "axis1": [ 655, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 297, 1 ], "axis1": [ 655, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 638, 8 ], "axis1": [ 53, 8 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 646, 2 ], "axis1": [ 265, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 648, 3 ], "axis1": [ 269, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 651, 3 ], "axis1": [ 290, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 654, 1 ], "axis1": [ 297, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 655, 8 ], "axis1": [ 53, 8 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 663, 2 ], "axis1": [ 265, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 665, 3 ], "axis1": [ 269, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 668, 3 ], "axis1": [ 290, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 671, 1 ], "axis1": [ 297, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 672, 4 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 106, 1 ], "axis1": [ 672, 2 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 267, 1 ], "axis1": [ 672, 2 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 674, 2 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 267, 1 ], "axis1": [ 674, 2 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 672, 1 ], "axis1": [ 106, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 673, 1 ], "axis1": [ 267, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 674, 1 ], "axis1": [ 106, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 675, 1 ], "axis1": [ 267, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 38, 34 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 26, 2 ], "axis1": [ 38, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 47, 1 ], "axis1": [ 38, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 38, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 51, 2 ], "axis1": [ 38, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 38, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 249, 2 ], "axis1": [ 38, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 252, 8 ], "axis1": [ 38, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 26, 2 ], "axis1": [ 55, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 47, 1 ], "axis1": [ 55, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 55, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 51, 2 ], "axis1": [ 55, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 55, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 249, 2 ], "axis1": [ 55, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 252, 8 ], "axis1": [ 55, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 26, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 40, 1 ], "axis1": [ 47, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 41, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 42, 2 ], "axis1": [ 51, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 44, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 45, 2 ], "axis1": [ 249, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 47, 8 ], "axis1": [ 252, 8 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 55, 2 ], "axis1": [ 26, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 57, 1 ], "axis1": [ 47, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 58, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 59, 2 ], "axis1": [ 51, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 61, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 62, 2 ], "axis1": [ 249, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 64, 8 ], "axis1": [ 252, 8 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 72, 34 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 45, 2 ], "axis1": [ 72, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 224, 3 ], "axis1": [ 72, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 228, 4 ], "axis1": [ 72, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 233, 8 ], "axis1": [ 72, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 45, 2 ], "axis1": [ 89, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 224, 3 ], "axis1": [ 89, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 228, 4 ], "axis1": [ 89, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 233, 8 ], "axis1": [ 89, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 72, 2 ], "axis1": [ 45, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 74, 3 ], "axis1": [ 224, 3 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 77, 4 ], "axis1": [ 228, 4 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 81, 8 ], "axis1": [ 233, 8 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 89, 2 ], "axis1": [ 45, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 91, 3 ], "axis1": [ 224, 3 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 94, 4 ], "axis1": [ 228, 4 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 98, 8 ], "axis1": [ 233, 8 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 106, 34 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 68, 1 ], "axis1": [ 106, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 76, 7 ], "axis1": [ 106, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 403, 1 ], "axis1": [ 106, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 405, 8 ], "axis1": [ 106, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 123, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 76, 7 ], "axis1": [ 123, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 403, 1 ], "axis1": [ 123, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 405, 8 ], "axis1": [ 123, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 68, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 107, 7 ], "axis1": [ 76, 7 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 114, 1 ], "axis1": [ 403, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 115, 8 ], "axis1": [ 405, 8 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 123, 1 ], "axis1": [ 68, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 124, 7 ], "axis1": [ 76, 7 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 131, 1 ], "axis1": [ 403, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 132, 8 ], "axis1": [ 405, 8 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 140, 34 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 260, 4 ], "axis1": [ 140, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 272, 13 ], "axis1": [ 140, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 260, 4 ], "axis1": [ 157, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 272, 13 ], "axis1": [ 157, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 140, 4 ], "axis1": [ 260, 4 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 144, 13 ], "axis1": [ 272, 13 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 157, 4 ], "axis1": [ 260, 4 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 161, 13 ], "axis1": [ 272, 13 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 174, 2 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 48, 1 ], "axis1": [ 174, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 48, 1 ], "axis1": [ 175, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 174, 1 ], "axis1": [ 48, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 175, 1 ], "axis1": [ 48, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 176, 2 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 176, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 177, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 176, 1 ], "axis1": [ 18, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 177, 1 ], "axis1": [ 18, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 178, 2 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 425, 1 ], "axis1": [ 178, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 425, 1 ], "axis1": [ 179, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 178, 1 ], "axis1": [ 425, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 179, 1 ], "axis1": [ 425, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 8, 34 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 53, 3 ], "axis1": [ 8, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 8, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 245, 4 ], "axis1": [ 8, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 265, 3 ], "axis1": [ 8, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 269, 3 ], "axis1": [ 8, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 287, 1 ], "axis1": [ 8, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 336, 1 ], "axis1": [ 8, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 391, 1 ], "axis1": [ 8, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 53, 3 ], "axis1": [ 25, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 25, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 245, 4 ], "axis1": [ 25, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 265, 3 ], "axis1": [ 25, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 269, 3 ], "axis1": [ 25, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 287, 1 ], "axis1": [ 25, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 336, 1 ], "axis1": [ 25, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 391, 1 ], "axis1": [ 25, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 8, 3 ], "axis1": [ 53, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 11, 1 ], "axis1": [ 67, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 12, 4 ], "axis1": [ 245, 4 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 16, 3 ], "axis1": [ 265, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 19, 3 ], "axis1": [ 269, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 22, 1 ], "axis1": [ 287, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 23, 1 ], "axis1": [ 336, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 391, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 25, 3 ], "axis1": [ 53, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 28, 1 ], "axis1": [ 67, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 29, 4 ], "axis1": [ 245, 4 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 33, 3 ], "axis1": [ 265, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 36, 3 ], "axis1": [ 269, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 39, 1 ], "axis1": [ 287, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 40, 1 ], "axis1": [ 336, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 41, 1 ], "axis1": [ 391, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 42, 34 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 33, 3 ], "axis1": [ 42, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 42, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 50, 1 ], "axis1": [ 42, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 166, 1 ], "axis1": [ 42, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 42, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 42, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 251, 1 ], "axis1": [ 42, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 42, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 42, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 33, 3 ], "axis1": [ 59, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 59, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 50, 1 ], "axis1": [ 59, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 166, 1 ], "axis1": [ 59, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 59, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 59, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 251, 1 ], "axis1": [ 59, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 59, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 59, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 42, 3 ], "axis1": [ 33, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 45, 2 ], "axis1": [ 38, 2 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 47, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 48, 1 ], "axis1": [ 166, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 49, 4 ], "axis1": [ 168, 4 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 53, 3 ], "axis1": [ 178, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 56, 1 ], "axis1": [ 251, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 57, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 58, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 59, 3 ], "axis1": [ 33, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 62, 2 ], "axis1": [ 38, 2 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 64, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 65, 1 ], "axis1": [ 166, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 66, 4 ], "axis1": [ 168, 4 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 70, 3 ], "axis1": [ 178, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 73, 1 ], "axis1": [ 251, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 74, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 75, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 76, 2 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 264, 1 ], "axis1": [ 76, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 264, 1 ], "axis1": [ 77, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 76, 1 ], "axis1": [ 264, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 77, 1 ], "axis1": [ 264, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 78, 2 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 56, 1 ], "axis1": [ 78, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 56, 1 ], "axis1": [ 79, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 78, 1 ], "axis1": [ 56, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 79, 1 ], "axis1": [ 56, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 13, 2 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 201, 1 ], "axis1": [ 13, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 201, 1 ], "axis1": [ 14, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 13, 1 ], "axis1": [ 201, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 14, 1 ], "axis1": [ 201, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 15, 2 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 204, 1 ], "axis1": [ 15, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 204, 1 ], "axis1": [ 16, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 15, 1 ], "axis1": [ 204, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 16, 1 ], "axis1": [ 204, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 17, 2 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 286, 1 ], "axis1": [ 17, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 286, 1 ], "axis1": [ 18, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 17, 1 ], "axis1": [ 286, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 18, 1 ], "axis1": [ 286, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 19, 2 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 202, 1 ], "axis1": [ 19, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 202, 1 ], "axis1": [ 20, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 19, 1 ], "axis1": [ 202, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 20, 1 ], "axis1": [ 202, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 21, 2 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 203, 1 ], "axis1": [ 21, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 203, 1 ], "axis1": [ 22, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 21, 1 ], "axis1": [ 203, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 22, 1 ], "axis1": [ 203, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 50, 1 ], "axis1": [ 5, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 50, 1 ], "axis1": [ 6, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 5, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 6, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 53, 1 ], "axis1": [ 7, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 53, 1 ], "axis1": [ 8, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 7, 1 ], "axis1": [ 53, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 8, 1 ], "axis1": [ 53, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 166, 1 ], "axis1": [ 9, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 166, 1 ], "axis1": [ 10, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 9, 1 ], "axis1": [ 166, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 10, 1 ], "axis1": [ 166, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 11, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 12, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 11, 1 ], "axis1": [ 33, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 12, 1 ], "axis1": [ 33, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 169, 1 ], "axis1": [ 3, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 169, 1 ], "axis1": [ 4, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 3, 1 ], "axis1": [ 169, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 4, 1 ], "axis1": [ 169, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1633, 34 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 1633, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 26, 2 ], "axis1": [ 1633, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 36, 1 ], "axis1": [ 1633, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 1633, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 76, 2 ], "axis1": [ 1633, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 172, 6 ], "axis1": [ 1633, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 181, 3 ], "axis1": [ 1633, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 187, 1 ], "axis1": [ 1633, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 1650, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 26, 2 ], "axis1": [ 1650, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 36, 1 ], "axis1": [ 1650, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 1650, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 76, 2 ], "axis1": [ 1650, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 172, 6 ], "axis1": [ 1650, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 181, 3 ], "axis1": [ 1650, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 187, 1 ], "axis1": [ 1650, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 1633, 1 ], "axis1": [ 18, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1634, 2 ], "axis1": [ 26, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1636, 1 ], "axis1": [ 36, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1637, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1638, 2 ], "axis1": [ 76, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1640, 6 ], "axis1": [ 172, 6 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1646, 3 ], "axis1": [ 181, 3 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1649, 1 ], "axis1": [ 187, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1650, 1 ], "axis1": [ 18, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1651, 2 ], "axis1": [ 26, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1653, 1 ], "axis1": [ 36, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1654, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1655, 2 ], "axis1": [ 76, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1657, 6 ], "axis1": [ 172, 6 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1663, 3 ], "axis1": [ 181, 3 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1666, 1 ], "axis1": [ 187, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1667, 34 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 19, 1 ], "axis1": [ 1667, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 1667, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 30, 2 ], "axis1": [ 1667, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 90, 6 ], "axis1": [ 1667, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 1667, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 153, 5 ], "axis1": [ 1667, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 1667, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 19, 1 ], "axis1": [ 1684, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 1684, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 30, 2 ], "axis1": [ 1684, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 90, 6 ], "axis1": [ 1684, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 1684, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 153, 5 ], "axis1": [ 1684, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 1684, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 1667, 1 ], "axis1": [ 19, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1668, 1 ], "axis1": [ 28, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1669, 2 ], "axis1": [ 30, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1671, 6 ], "axis1": [ 90, 6 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1677, 1 ], "axis1": [ 106, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1678, 5 ], "axis1": [ 153, 5 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1683, 1 ], "axis1": [ 331, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1684, 1 ], "axis1": [ 19, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1685, 1 ], "axis1": [ 28, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1686, 2 ], "axis1": [ 30, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1688, 6 ], "axis1": [ 90, 6 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1694, 1 ], "axis1": [ 106, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1695, 5 ], "axis1": [ 153, 5 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1700, 1 ], "axis1": [ 331, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1701, 34 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 21, 3 ], "axis1": [ 1701, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 1701, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 1701, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 1701, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 1701, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 21, 3 ], "axis1": [ 1718, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 1718, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 113, 5 ], "axis1": [ 1718, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 120, 7 ], "axis1": [ 1718, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 1718, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 1701, 3 ], "axis1": [ 21, 3 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1704, 1 ], "axis1": [ 111, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1705, 5 ], "axis1": [ 113, 5 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1710, 7 ], "axis1": [ 120, 7 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1717, 1 ], "axis1": [ 330, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1718, 3 ], "axis1": [ 21, 3 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1721, 1 ], "axis1": [ 111, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1722, 5 ], "axis1": [ 113, 5 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1727, 7 ], "axis1": [ 120, 7 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1734, 1 ], "axis1": [ 330, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1735, 34 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 37, 1 ], "axis1": [ 1735, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 184, 3 ], "axis1": [ 1735, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 188, 13 ], "axis1": [ 1735, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 37, 1 ], "axis1": [ 1752, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 184, 3 ], "axis1": [ 1752, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 188, 13 ], "axis1": [ 1752, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 1735, 1 ], "axis1": [ 37, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1736, 3 ], "axis1": [ 184, 3 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1739, 13 ], "axis1": [ 188, 13 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1752, 1 ], "axis1": [ 37, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1753, 3 ], "axis1": [ 184, 3 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1756, 13 ], "axis1": [ 188, 13 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1769, 34 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 32, 1 ], "axis1": [ 1769, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 158, 8 ], "axis1": [ 1769, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 207, 5 ], "axis1": [ 1769, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 1769, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 1769, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 1769, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 32, 1 ], "axis1": [ 1786, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 158, 8 ], "axis1": [ 1786, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 207, 5 ], "axis1": [ 1786, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 1786, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 1786, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 1786, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 1769, 1 ], "axis1": [ 32, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1770, 8 ], "axis1": [ 158, 8 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1778, 5 ], "axis1": [ 207, 5 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1783, 1 ], "axis1": [ 214, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1784, 1 ], "axis1": [ 333, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1785, 1 ], "axis1": [ 392, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1786, 1 ], "axis1": [ 32, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1787, 8 ], "axis1": [ 158, 8 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1795, 5 ], "axis1": [ 207, 5 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1800, 1 ], "axis1": [ 214, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1801, 1 ], "axis1": [ 333, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1802, 1 ], "axis1": [ 392, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1803, 34 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 426, 17 ], "axis1": [ 1803, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 426, 17 ], "axis1": [ 1820, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 1803, 17 ], "axis1": [ 426, 17 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1820, 17 ], "axis1": [ 426, 17 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1837, 34 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 24, 2 ], "axis1": [ 1837, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 2 ], "axis1": [ 1837, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 107, 2 ], "axis1": [ 1837, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 1837, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 112, 1 ], "axis1": [ 1837, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 127, 9 ], "axis1": [ 1837, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 24, 2 ], "axis1": [ 1854, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 40, 2 ], "axis1": [ 1854, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 107, 2 ], "axis1": [ 1854, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 1854, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 112, 1 ], "axis1": [ 1854, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 127, 9 ], "axis1": [ 1854, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 1837, 2 ], "axis1": [ 24, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1839, 2 ], "axis1": [ 40, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1841, 2 ], "axis1": [ 107, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1843, 1 ], "axis1": [ 110, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1844, 1 ], "axis1": [ 112, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1845, 9 ], "axis1": [ 127, 9 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1854, 2 ], "axis1": [ 24, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1856, 2 ], "axis1": [ 40, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1858, 2 ], "axis1": [ 107, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1860, 1 ], "axis1": [ 110, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1861, 1 ], "axis1": [ 112, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1862, 9 ], "axis1": [ 127, 9 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1871, 34 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 136, 17 ], "axis1": [ 1871, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 136, 17 ], "axis1": [ 1888, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 1871, 17 ], "axis1": [ 136, 17 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1888, 17 ], "axis1": [ 136, 17 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1905, 34 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 38, 2 ], "axis1": [ 1905, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 1905, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 1905, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 201, 6 ], "axis1": [ 1905, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 213, 1 ], "axis1": [ 1905, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 215, 1 ], "axis1": [ 1905, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1905, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 1905, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1905, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 1922, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 1922, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 178, 3 ], "axis1": [ 1922, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 201, 6 ], "axis1": [ 1922, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 213, 1 ], "axis1": [ 1922, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 215, 1 ], "axis1": [ 1922, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1922, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 1922, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1922, 17 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 1905, 2 ], "axis1": [ 38, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1907, 1 ], "axis1": [ 42, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1908, 3 ], "axis1": [ 178, 3 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1911, 6 ], "axis1": [ 201, 6 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1917, 1 ], "axis1": [ 213, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1918, 1 ], "axis1": [ 215, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1919, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1920, 1 ], "axis1": [ 334, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1921, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1922, 2 ], "axis1": [ 38, 2 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1924, 1 ], "axis1": [ 42, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1925, 3 ], "axis1": [ 178, 3 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1928, 6 ], "axis1": [ 201, 6 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1934, 1 ], "axis1": [ 213, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1935, 1 ], "axis1": [ 215, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1936, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1937, 1 ], "axis1": [ 334, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1938, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1939, 24 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 33, 3 ], "axis1": [ 1939, 12 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 1939, 12 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 166, 1 ], "axis1": [ 1939, 12 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 1939, 12 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 33, 3 ], "axis1": [ 1951, 12 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 1951, 12 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 166, 1 ], "axis1": [ 1951, 12 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 168, 4 ], "axis1": [ 1951, 12 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 1939, 3 ], "axis1": [ 33, 3 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1942, 4 ], "axis1": [ 53, 4 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1946, 1 ], "axis1": [ 166, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1947, 4 ], "axis1": [ 168, 4 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1951, 3 ], "axis1": [ 33, 3 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1954, 4 ], "axis1": [ 53, 4 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1958, 1 ], "axis1": [ 166, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 1959, 4 ], "axis1": [ 168, 4 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 137, 408 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 43, 10 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 61, 5 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 67, 2 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 70, 6 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 78, 11 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 216, 1 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 218, 4 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 224, 8 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 233, 4 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 238, 4 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 243, 4 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 252, 10 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 263, 1 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 265, 3 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 269, 16 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 286, 2 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 301, 6 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 313, 4 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 318, 1 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 328, 2 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 332, 1 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 335, 2 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 338, 1 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 346, 12 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 359, 4 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 364, 3 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 369, 9 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 379, 13 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 394, 9 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 405, 7 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 414, 1 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 416, 10 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 443, 15 ], "axis1": [ 137, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 43, 10 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 61, 5 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 67, 2 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 70, 6 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 78, 11 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 216, 1 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 218, 4 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 224, 8 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 233, 4 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 238, 4 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 243, 4 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 252, 10 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 263, 1 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 265, 3 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 269, 16 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 286, 2 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 301, 6 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 313, 4 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 318, 1 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 328, 2 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 332, 1 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 335, 2 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 338, 1 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 346, 12 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 359, 4 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 364, 3 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 369, 9 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 379, 13 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 394, 9 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 405, 7 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 414, 1 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 416, 10 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 443, 15 ], "axis1": [ 341, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 137, 10 ], "axis1": [ 43, 10 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 147, 5 ], "axis1": [ 61, 5 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 152, 2 ], "axis1": [ 67, 2 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 154, 6 ], "axis1": [ 70, 6 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 160, 11 ], "axis1": [ 78, 11 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 171, 1 ], "axis1": [ 216, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 172, 4 ], "axis1": [ 218, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 176, 8 ], "axis1": [ 224, 8 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 184, 4 ], "axis1": [ 233, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 188, 4 ], "axis1": [ 238, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 192, 4 ], "axis1": [ 243, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 196, 3 ], "axis1": [ 248, 3 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 199, 10 ], "axis1": [ 252, 10 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 209, 1 ], "axis1": [ 263, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 210, 3 ], "axis1": [ 265, 3 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 213, 16 ], "axis1": [ 269, 16 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 229, 2 ], "axis1": [ 286, 2 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 231, 6 ], "axis1": [ 301, 6 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 237, 4 ], "axis1": [ 308, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 241, 4 ], "axis1": [ 313, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 245, 1 ], "axis1": [ 318, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 246, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 249, 3 ], "axis1": [ 324, 3 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 252, 2 ], "axis1": [ 328, 2 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 254, 1 ], "axis1": [ 332, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 255, 2 ], "axis1": [ 335, 2 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 257, 1 ], "axis1": [ 338, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 258, 12 ], "axis1": [ 346, 12 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 270, 4 ], "axis1": [ 359, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 274, 3 ], "axis1": [ 364, 3 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 277, 9 ], "axis1": [ 369, 9 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 286, 13 ], "axis1": [ 379, 13 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 299, 9 ], "axis1": [ 394, 9 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 308, 7 ], "axis1": [ 405, 7 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 315, 1 ], "axis1": [ 414, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 316, 10 ], "axis1": [ 416, 10 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 326, 15 ], "axis1": [ 443, 15 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 341, 10 ], "axis1": [ 43, 10 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 351, 5 ], "axis1": [ 61, 5 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 356, 2 ], "axis1": [ 67, 2 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 358, 6 ], "axis1": [ 70, 6 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 364, 11 ], "axis1": [ 78, 11 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 375, 1 ], "axis1": [ 216, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 376, 4 ], "axis1": [ 218, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 380, 8 ], "axis1": [ 224, 8 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 388, 4 ], "axis1": [ 233, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 392, 4 ], "axis1": [ 238, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 396, 4 ], "axis1": [ 243, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 400, 3 ], "axis1": [ 248, 3 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 403, 10 ], "axis1": [ 252, 10 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 413, 1 ], "axis1": [ 263, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 414, 3 ], "axis1": [ 265, 3 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 417, 16 ], "axis1": [ 269, 16 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 433, 2 ], "axis1": [ 286, 2 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 435, 6 ], "axis1": [ 301, 6 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 441, 4 ], "axis1": [ 308, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 445, 4 ], "axis1": [ 313, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 449, 1 ], "axis1": [ 318, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 450, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 453, 3 ], "axis1": [ 324, 3 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 456, 2 ], "axis1": [ 328, 2 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 458, 1 ], "axis1": [ 332, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 459, 2 ], "axis1": [ 335, 2 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 461, 1 ], "axis1": [ 338, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 462, 12 ], "axis1": [ 346, 12 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 474, 4 ], "axis1": [ 359, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 478, 3 ], "axis1": [ 364, 3 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 481, 9 ], "axis1": [ 369, 9 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 490, 13 ], "axis1": [ 379, 13 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 503, 9 ], "axis1": [ 394, 9 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 512, 7 ], "axis1": [ 405, 7 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 519, 1 ], "axis1": [ 414, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 520, 10 ], "axis1": [ 416, 10 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 530, 15 ], "axis1": [ 443, 15 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 545, 408 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 20, 1 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 97, 4 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 344, 2 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 458, 6 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 465, 14 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 480, 9 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 490, 4 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 495, 4 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 500, 4 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 505, 1 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 507, 6 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 515, 4 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 520, 1 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 530, 1 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 533, 138 ], "axis1": [ 545, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 20, 1 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 97, 4 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 344, 2 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 458, 6 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 465, 14 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 480, 9 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 490, 4 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 495, 4 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 500, 4 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 505, 1 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 507, 6 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 515, 4 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 520, 1 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 530, 1 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 533, 138 ], "axis1": [ 749, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 545, 1 ], "axis1": [ 20, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 546, 1 ], "axis1": [ 69, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 547, 4 ], "axis1": [ 97, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 551, 4 ], "axis1": [ 102, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 555, 2 ], "axis1": [ 344, 2 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 557, 6 ], "axis1": [ 458, 6 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 563, 14 ], "axis1": [ 465, 14 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 577, 9 ], "axis1": [ 480, 9 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 586, 4 ], "axis1": [ 490, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 590, 4 ], "axis1": [ 495, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 594, 4 ], "axis1": [ 500, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 598, 1 ], "axis1": [ 505, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 599, 6 ], "axis1": [ 507, 6 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 605, 4 ], "axis1": [ 515, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 609, 1 ], "axis1": [ 520, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 610, 1 ], "axis1": [ 530, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 611, 138 ], "axis1": [ 533, 138 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 749, 1 ], "axis1": [ 20, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 750, 1 ], "axis1": [ 69, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 751, 4 ], "axis1": [ 97, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 755, 4 ], "axis1": [ 102, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 759, 2 ], "axis1": [ 344, 2 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 761, 6 ], "axis1": [ 458, 6 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 767, 14 ], "axis1": [ 465, 14 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 781, 9 ], "axis1": [ 480, 9 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 790, 4 ], "axis1": [ 490, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 794, 4 ], "axis1": [ 495, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 798, 4 ], "axis1": [ 500, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 802, 1 ], "axis1": [ 505, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 803, 6 ], "axis1": [ 507, 6 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 809, 4 ], "axis1": [ 515, 4 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 813, 1 ], "axis1": [ 520, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 814, 1 ], "axis1": [ 530, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 815, 138 ], "axis1": [ 533, 138 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 953, 408 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 671, 204 ], "axis1": [ 953, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 671, 204 ], "axis1": [ 1157, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 953, 204 ], "axis1": [ 671, 204 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 1157, 204 ], "axis1": [ 671, 204 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1361, 408 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 875, 204 ], "axis1": [ 1361, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 875, 204 ], "axis1": [ 1565, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 1361, 204 ], "axis1": [ 875, 204 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 1565, 204 ], "axis1": [ 875, 204 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1769, 408 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 1079, 204 ], "axis1": [ 1769, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 1079, 204 ], "axis1": [ 1973, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 1769, 204 ], "axis1": [ 1079, 204 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 1973, 204 ], "axis1": [ 1079, 204 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2177, 408 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 1283, 204 ], "axis1": [ 2177, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 1283, 204 ], "axis1": [ 2381, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 2177, 204 ], "axis1": [ 1283, 204 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 2381, 204 ], "axis1": [ 1283, 204 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2585, 408 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 1487, 204 ], "axis1": [ 2585, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 1487, 204 ], "axis1": [ 2789, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 2585, 204 ], "axis1": [ 1487, 204 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 2789, 204 ], "axis1": [ 1487, 204 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2993, 408 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 1691, 204 ], "axis1": [ 2993, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 1691, 204 ], "axis1": [ 3197, 204 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 2993, 204 ], "axis1": [ 1691, 204 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 3197, 204 ], "axis1": [ 1691, 204 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3401, 2 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 339, 1 ], "axis1": [ 3401, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 3402, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 3401, 1 ], "axis1": [ 339, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 3402, 1 ], "axis1": [ 339, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3403, 2 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 340, 1 ], "axis1": [ 3403, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 340, 1 ], "axis1": [ 3404, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 3403, 1 ], "axis1": [ 340, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 3404, 1 ], "axis1": [ 340, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3405, 2 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 293, 1 ], "axis1": [ 3405, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 293, 1 ], "axis1": [ 3406, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 3405, 1 ], "axis1": [ 293, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 3406, 1 ], "axis1": [ 293, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3407, 2 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 294, 1 ], "axis1": [ 3407, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 294, 1 ], "axis1": [ 3408, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 3407, 1 ], "axis1": [ 294, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 3408, 1 ], "axis1": [ 294, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3409, 2 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 342, 1 ], "axis1": [ 3409, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 342, 1 ], "axis1": [ 3410, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 3409, 1 ], "axis1": [ 342, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 3410, 1 ], "axis1": [ 342, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3411, 2 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 295, 1 ], "axis1": [ 3411, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 295, 1 ], "axis1": [ 3412, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 3411, 1 ], "axis1": [ 295, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 3412, 1 ], "axis1": [ 295, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3413, 2 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 343, 1 ], "axis1": [ 3413, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 343, 1 ], "axis1": [ 3414, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 3413, 1 ], "axis1": [ 343, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 3414, 1 ], "axis1": [ 343, 1 ], "diagonal": true, "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 426, 17 ], "axis1": [ 35, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 426, 17 ], "axis1": [ 52, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 35, 17 ], "axis1": [ 426, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 52, 17 ], "axis1": [ 426, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 18, 2 ], "axis1": [ 69, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 21, 2 ], "axis1": [ 69, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 6 ], "axis1": [ 69, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 106, 3 ], "axis1": [ 69, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 110, 4 ], "axis1": [ 69, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 18, 2 ], "axis1": [ 86, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 21, 2 ], "axis1": [ 86, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 6 ], "axis1": [ 86, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 106, 3 ], "axis1": [ 86, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 110, 4 ], "axis1": [ 86, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 2 ], "axis1": [ 18, 2 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 71, 2 ], "axis1": [ 21, 2 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 73, 6 ], "axis1": [ 90, 6 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 79, 3 ], "axis1": [ 106, 3 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 82, 4 ], "axis1": [ 110, 4 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 86, 2 ], "axis1": [ 18, 2 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 88, 2 ], "axis1": [ 21, 2 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 90, 6 ], "axis1": [ 90, 6 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 96, 3 ], "axis1": [ 106, 3 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 99, 4 ], "axis1": [ 110, 4 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 103, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 23, 3 ], "axis1": [ 103, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 114, 4 ], "axis1": [ 103, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 120, 9 ], "axis1": [ 103, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 103, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 23, 3 ], "axis1": [ 120, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 114, 4 ], "axis1": [ 120, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 120, 9 ], "axis1": [ 120, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 120, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 103, 3 ], "axis1": [ 23, 3 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 106, 4 ], "axis1": [ 114, 4 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 110, 9 ], "axis1": [ 120, 9 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 119, 1 ], "axis1": [ 330, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 120, 3 ], "axis1": [ 23, 3 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 123, 4 ], "axis1": [ 114, 4 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 127, 9 ], "axis1": [ 120, 9 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 136, 1 ], "axis1": [ 330, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 137, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 26, 3 ], "axis1": [ 137, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 137, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 129, 12 ], "axis1": [ 137, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 137, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 26, 3 ], "axis1": [ 154, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 154, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 129, 12 ], "axis1": [ 154, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 154, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 137, 3 ], "axis1": [ 26, 3 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 140, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 141, 12 ], "axis1": [ 129, 12 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 153, 1 ], "axis1": [ 331, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 154, 3 ], "axis1": [ 26, 3 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 157, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 158, 12 ], "axis1": [ 129, 12 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 170, 1 ], "axis1": [ 331, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 171, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 34, 3 ], "axis1": [ 171, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 172, 13 ], "axis1": [ 171, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 171, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 34, 3 ], "axis1": [ 188, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 172, 13 ], "axis1": [ 188, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 188, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 171, 3 ], "axis1": [ 34, 3 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 174, 13 ], "axis1": [ 172, 13 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 187, 1 ], "axis1": [ 392, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 188, 3 ], "axis1": [ 34, 3 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 191, 13 ], "axis1": [ 172, 13 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 204, 1 ], "axis1": [ 392, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 205, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 40, 2 ], "axis1": [ 205, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 197, 14 ], "axis1": [ 205, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 205, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 40, 2 ], "axis1": [ 222, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 197, 14 ], "axis1": [ 222, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 222, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 205, 2 ], "axis1": [ 40, 2 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 207, 14 ], "axis1": [ 197, 14 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 221, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 222, 2 ], "axis1": [ 40, 2 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 224, 14 ], "axis1": [ 197, 14 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 238, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 239, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1900, 17 ], "axis1": [ 239, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1900, 17 ], "axis1": [ 256, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 239, 17 ], "axis1": [ 1900, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 256, 17 ], "axis1": [ 1900, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 273, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1917, 17 ], "axis1": [ 273, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1917, 17 ], "axis1": [ 290, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 273, 17 ], "axis1": [ 1917, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 290, 17 ], "axis1": [ 1917, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 307, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1934, 17 ], "axis1": [ 307, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1934, 17 ], "axis1": [ 324, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 307, 17 ], "axis1": [ 1934, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 324, 17 ], "axis1": [ 1934, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 341, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1951, 17 ], "axis1": [ 341, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1951, 17 ], "axis1": [ 358, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 341, 17 ], "axis1": [ 1951, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 358, 17 ], "axis1": [ 1951, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 375, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1968, 17 ], "axis1": [ 375, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1968, 17 ], "axis1": [ 392, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 375, 17 ], "axis1": [ 1968, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 392, 17 ], "axis1": [ 1968, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 409, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1985, 17 ], "axis1": [ 409, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1985, 17 ], "axis1": [ 426, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 409, 17 ], "axis1": [ 1985, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 426, 17 ], "axis1": [ 1985, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 443, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 57, 4 ], "axis1": [ 443, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 285, 1 ], "axis1": [ 443, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 5 ], "axis1": [ 443, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 296, 5 ], "axis1": [ 443, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 443, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 393, 1 ], "axis1": [ 443, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 57, 4 ], "axis1": [ 460, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 285, 1 ], "axis1": [ 460, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 5 ], "axis1": [ 460, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 296, 5 ], "axis1": [ 460, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 460, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 393, 1 ], "axis1": [ 460, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 443, 4 ], "axis1": [ 57, 4 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 447, 1 ], "axis1": [ 285, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 448, 5 ], "axis1": [ 288, 5 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 453, 5 ], "axis1": [ 296, 5 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 458, 1 ], "axis1": [ 337, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 459, 1 ], "axis1": [ 393, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 460, 4 ], "axis1": [ 57, 4 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 464, 1 ], "axis1": [ 285, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 465, 5 ], "axis1": [ 288, 5 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 470, 5 ], "axis1": [ 296, 5 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 475, 1 ], "axis1": [ 337, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 476, 1 ], "axis1": [ 393, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 477, 2 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 29, 1 ], "axis1": [ 477, 1 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 29, 1 ], "axis1": [ 478, 1 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 477, 1 ], "axis1": [ 29, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 478, 1 ], "axis1": [ 29, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" } ] } }, "reference_fixtures": { "included": [], "reason": "No input corpus was supplied to compile_hf_bundle.", "scope": "Input-specific activations and intervention results." }, "residual_stream": { "constant_one_column": 0, "constant_one_semantics": "Holds exactly 1.0 at every position. Read as the \u0394=0 self-match query/key source by every attention operation except compute_attn (their query/key source columns) and, under bias=False, by MLP bias folds (their W_in/W_up rows in physical_layout.placements). Patching it collapses every self-match head.", "rms_norm_reserved_columns": [ 2046, 2047 ], "state_semantics": "input and post-layer residual streams; the compiler has no stable observation boundary between attention and MLP", "states": [ { "key": "input", "nodes": { "i:0": [ [ 0, 1 ] ], "l:8": [ [ 1, 17 ] ] } }, { "key": "L0.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:102": [ [ 27, 1 ] ], "l:103": [ [ 28, 1 ] ], "l:105": [ [ 22, 1 ] ], "l:106": [ [ 29, 1 ] ], "l:114": [ [ 23, 1 ] ], "l:118": [ [ 35, 1 ] ], "l:119": [ [ 36, 1 ] ], "l:121": [ [ 37, 1 ] ], "l:127": [ [ 52, 1 ] ], "l:130": [ [ 38, 4 ] ], "l:133": [ [ 53, 17 ] ], "l:136": [ [ 97, 1 ] ], "l:137": [ [ 98, 1 ] ], "l:139": [ [ 99, 1 ] ], "l:144": [ [ 49, 1 ] ], "l:145": [ [ 50, 1 ] ], "l:147": [ [ 51, 1 ] ], "l:151": [ [ 34, 1 ] ], "l:154": [ [ 30, 1 ] ], "l:155": [ [ 31, 1 ] ], "l:157": [ [ 32, 1 ] ], "l:171": [ [ 70, 1 ] ], "l:174": [ [ 42, 4 ] ], "l:177": [ [ 71, 17 ] ], "l:186": [ [ 100, 1 ] ], "l:187": [ [ 101, 1 ] ], "l:189": [ [ 102, 1 ] ], "l:196": [ [ 91, 1 ] ], "l:197": [ [ 92, 1 ] ], "l:199": [ [ 93, 1 ] ], "l:210": [ [ 103, 1 ] ], "l:211": [ [ 104, 1 ] ], "l:213": [ [ 105, 1 ] ], "l:222": [ [ 94, 1 ] ], "l:223": [ [ 95, 1 ] ], "l:225": [ [ 96, 1 ] ], "l:242": [ [ 106, 1 ] ], "l:4": [ [ 107, 1 ] ], "l:46": [ [ 18, 1 ] ], "l:48": [ [ 19, 1 ] ], "l:5": [ [ 108, 1 ] ], "l:50": [ [ 20, 1 ] ], "l:54": [ [ 24, 1 ] ], "l:55": [ [ 25, 1 ] ], "l:57": [ [ 26, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:8": [ [ 1, 17 ] ], "l:84": [ [ 88, 1 ] ], "l:85": [ [ 89, 1 ] ], "l:87": [ [ 90, 1 ] ], "l:9": [ [ 109, 1 ] ], "l:92": [ [ 46, 1 ] ], "l:93": [ [ 47, 1 ] ], "l:95": [ [ 48, 1 ] ], "l:99": [ [ 33, 1 ] ] } }, { "key": "L1.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:100": [ [ 28, 1 ] ], "l:107": [ [ 117, 1 ] ], "l:114": [ [ 23, 1 ] ], "l:118": [ [ 35, 1 ] ], "l:119": [ [ 36, 1 ] ], "l:121": [ [ 37, 1 ] ], "l:127": [ [ 52, 1 ] ], "l:128": [ [ 113, 4 ] ], "l:133": [ [ 53, 17 ] ], "l:136": [ [ 97, 1 ] ], "l:137": [ [ 98, 1 ] ], "l:139": [ [ 99, 1 ] ], "l:144": [ [ 49, 1 ] ], "l:145": [ [ 50, 1 ] ], "l:147": [ [ 51, 1 ] ], "l:151": [ [ 34, 1 ] ], "l:154": [ [ 30, 1 ] ], "l:155": [ [ 31, 1 ] ], "l:157": [ [ 32, 1 ] ], "l:165": [ [ 112, 1 ] ], "l:171": [ [ 70, 1 ] ], "l:174": [ [ 42, 4 ] ], "l:177": [ [ 71, 17 ] ], "l:186": [ [ 100, 1 ] ], "l:187": [ [ 101, 1 ] ], "l:189": [ [ 102, 1 ] ], "l:196": [ [ 91, 1 ] ], "l:197": [ [ 92, 1 ] ], "l:199": [ [ 93, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:20": [ [ 119, 1 ] ], "l:21": [ [ 118, 1 ] ], "l:210": [ [ 103, 1 ] ], "l:211": [ [ 104, 1 ] ], "l:213": [ [ 105, 1 ] ], "l:222": [ [ 94, 1 ] ], "l:223": [ [ 95, 1 ] ], "l:225": [ [ 96, 1 ] ], "l:242": [ [ 106, 1 ] ], "l:45": [ [ 108, 1 ] ], "l:46": [ [ 18, 1 ] ], "l:47": [ [ 107, 1 ] ], "l:48": [ [ 19, 1 ] ], "l:49": [ [ 110, 1 ] ], "l:50": [ [ 20, 1 ] ], "l:51": [ [ 111, 1 ] ], "l:52": [ [ 27, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:8": [ [ 1, 17 ] ], "l:84": [ [ 88, 1 ] ], "l:85": [ [ 89, 1 ] ], "l:87": [ [ 90, 1 ] ], "l:92": [ [ 46, 1 ] ], "l:93": [ [ 47, 1 ] ], "l:95": [ [ 48, 1 ] ], "l:99": [ [ 33, 1 ] ] } }, { "key": "L2.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:114": [ [ 23, 1 ] ], "l:118": [ [ 35, 1 ] ], "l:119": [ [ 36, 1 ] ], "l:121": [ [ 37, 1 ] ], "l:127": [ [ 52, 1 ] ], "l:128": [ [ 113, 4 ] ], "l:133": [ [ 53, 17 ] ], "l:136": [ [ 97, 1 ] ], "l:137": [ [ 98, 1 ] ], "l:139": [ [ 99, 1 ] ], "l:144": [ [ 49, 1 ] ], "l:145": [ [ 50, 1 ] ], "l:147": [ [ 51, 1 ] ], "l:151": [ [ 34, 1 ] ], "l:154": [ [ 30, 1 ] ], "l:155": [ [ 31, 1 ] ], "l:157": [ [ 32, 1 ] ], "l:165": [ [ 112, 1 ] ], "l:171": [ [ 70, 1 ] ], "l:174": [ [ 42, 4 ] ], "l:177": [ [ 71, 17 ] ], "l:18": [ [ 28, 1 ] ], "l:186": [ [ 100, 1 ] ], "l:187": [ [ 101, 1 ] ], "l:189": [ [ 102, 1 ] ], "l:19": [ [ 25, 1 ] ], "l:196": [ [ 91, 1 ] ], "l:197": [ [ 92, 1 ] ], "l:199": [ [ 93, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:210": [ [ 103, 1 ] ], "l:211": [ [ 104, 1 ] ], "l:213": [ [ 105, 1 ] ], "l:222": [ [ 94, 1 ] ], "l:223": [ [ 95, 1 ] ], "l:225": [ [ 96, 1 ] ], "l:242": [ [ 106, 1 ] ], "l:43": [ [ 26, 1 ] ], "l:46": [ [ 18, 1 ] ], "l:48": [ [ 19, 1 ] ], "l:50": [ [ 20, 1 ] ], "l:51": [ [ 111, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:8": [ [ 1, 17 ] ], "l:84": [ [ 88, 1 ] ], "l:85": [ [ 89, 1 ] ], "l:87": [ [ 90, 1 ] ], "l:92": [ [ 46, 1 ] ], "l:93": [ [ 47, 1 ] ], "l:95": [ [ 48, 1 ] ], "l:97": [ [ 27, 1 ] ], "l:98": [ [ 24, 1 ] ] } }, { "key": "L3.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:114": [ [ 23, 1 ] ], "l:118": [ [ 35, 1 ] ], "l:119": [ [ 36, 1 ] ], "l:121": [ [ 37, 1 ] ], "l:125": [ [ 108, 1 ] ], "l:127": [ [ 52, 1 ] ], "l:128": [ [ 113, 4 ] ], "l:133": [ [ 53, 17 ] ], "l:136": [ [ 97, 1 ] ], "l:137": [ [ 98, 1 ] ], "l:139": [ [ 99, 1 ] ], "l:144": [ [ 49, 1 ] ], "l:145": [ [ 50, 1 ] ], "l:147": [ [ 51, 1 ] ], "l:151": [ [ 34, 1 ] ], "l:154": [ [ 30, 1 ] ], "l:155": [ [ 31, 1 ] ], "l:157": [ [ 32, 1 ] ], "l:165": [ [ 112, 1 ] ], "l:17": [ [ 25, 1 ] ], "l:171": [ [ 70, 1 ] ], "l:174": [ [ 42, 4 ] ], "l:177": [ [ 71, 17 ] ], "l:186": [ [ 100, 1 ] ], "l:187": [ [ 101, 1 ] ], "l:189": [ [ 102, 1 ] ], "l:196": [ [ 91, 1 ] ], "l:197": [ [ 92, 1 ] ], "l:199": [ [ 93, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:210": [ [ 103, 1 ] ], "l:211": [ [ 104, 1 ] ], "l:213": [ [ 105, 1 ] ], "l:222": [ [ 94, 1 ] ], "l:223": [ [ 95, 1 ] ], "l:225": [ [ 96, 1 ] ], "l:242": [ [ 106, 1 ] ], "l:376": [ [ 127, 17 ] ], "l:379": [ [ 144, 17 ] ], "l:38": [ [ 117, 1 ] ], "l:382": [ [ 161, 17 ] ], "l:387": [ [ 38, 1 ] ], "l:388": [ [ 39, 1 ] ], "l:39": [ [ 22, 1 ] ], "l:41": [ [ 33, 1 ] ], "l:415": [ [ 40, 1 ] ], "l:416": [ [ 41, 1 ] ], "l:435": [ [ 110, 1 ] ], "l:436": [ [ 107, 1 ] ], "l:46": [ [ 18, 1 ] ], "l:48": [ [ 19, 1 ] ], "l:50": [ [ 20, 1 ] ], "l:58": [ [ 120, 1 ] ], "l:64": [ [ 121, 1 ] ], "l:65": [ [ 122, 1 ] ], "l:67": [ [ 123, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:73": [ [ 124, 1 ] ], "l:74": [ [ 125, 1 ] ], "l:76": [ [ 126, 1 ] ], "l:8": [ [ 1, 17 ] ], "l:84": [ [ 88, 1 ] ], "l:85": [ [ 89, 1 ] ], "l:87": [ [ 90, 1 ] ], "l:92": [ [ 46, 1 ] ], "l:93": [ [ 47, 1 ] ], "l:95": [ [ 48, 1 ] ], "l:96": [ [ 24, 1 ] ] } }, { "key": "L4.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:114": [ [ 23, 1 ] ], "l:115": [ [ 20, 1 ] ], "l:116": [ [ 37, 1 ] ], "l:123": [ [ 22, 1 ], [ 111, 1 ], [ 117, 1 ] ], "l:126": [ [ 19, 1 ] ], "l:127": [ [ 52, 1 ] ], "l:128": [ [ 113, 4 ] ], "l:133": [ [ 53, 17 ] ], "l:136": [ [ 97, 1 ] ], "l:137": [ [ 98, 1 ] ], "l:139": [ [ 99, 1 ] ], "l:144": [ [ 49, 1 ] ], "l:145": [ [ 50, 1 ] ], "l:147": [ [ 51, 1 ] ], "l:151": [ [ 34, 1 ] ], "l:152": [ [ 36, 1 ] ], "l:158": [ [ 296, 1 ] ], "l:16": [ [ 283, 1 ] ], "l:163": [ [ 120, 1 ] ], "l:171": [ [ 70, 1 ] ], "l:172": [ [ 121, 4 ] ], "l:177": [ [ 71, 17 ] ], "l:186": [ [ 100, 1 ] ], "l:187": [ [ 101, 1 ] ], "l:189": [ [ 102, 1 ] ], "l:196": [ [ 91, 1 ] ], "l:197": [ [ 92, 1 ] ], "l:199": [ [ 93, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:201": [ [ 28, 1 ], [ 30, 3 ] ], "l:210": [ [ 103, 1 ] ], "l:211": [ [ 104, 1 ] ], "l:213": [ [ 105, 1 ] ], "l:22": [ [ 284, 1 ] ], "l:222": [ [ 94, 1 ] ], "l:223": [ [ 95, 1 ] ], "l:225": [ [ 96, 1 ] ], "l:227": [ [ 26, 2 ], [ 125, 2 ] ], "l:242": [ [ 106, 1 ] ], "l:36": [ [ 47, 1 ] ], "l:366": [ [ 285, 1 ] ], "l:367": [ [ 286, 1 ] ], "l:374": [ [ 181, 17 ] ], "l:377": [ [ 198, 17 ] ], "l:380": [ [ 215, 17 ] ], "l:385": [ [ 293, 1 ] ], "l:387": [ [ 38, 1 ] ], "l:388": [ [ 39, 1 ] ], "l:395": [ [ 232, 17 ] ], "l:397": [ [ 249, 17 ] ], "l:399": [ [ 266, 17 ] ], "l:405": [ [ 287, 1 ] ], "l:406": [ [ 288, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:415": [ [ 40, 1 ] ], "l:416": [ [ 41, 1 ] ], "l:425": [ [ 289, 1 ] ], "l:426": [ [ 290, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:435": [ [ 110, 1 ] ], "l:436": [ [ 107, 1 ] ], "l:445": [ [ 291, 1 ] ], "l:446": [ [ 292, 1 ] ], "l:59": [ [ 297, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:68": [ [ 314, 17 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:8": [ [ 1, 17 ] ], "l:84": [ [ 88, 1 ] ], "l:85": [ [ 89, 1 ] ], "l:87": [ [ 90, 1 ] ], "l:89": [ [ 35, 1 ], [ 178, 3 ] ], "l:90": [ [ 46, 1 ] ] } }, { "key": "L5.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:112": [ [ 154, 1 ] ], "l:122": [ [ 34, 1 ], [ 38, 2 ] ], "l:127": [ [ 52, 1 ] ], "l:128": [ [ 113, 4 ] ], "l:133": [ [ 53, 17 ] ], "l:136": [ [ 97, 1 ] ], "l:137": [ [ 98, 1 ] ], "l:139": [ [ 99, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:144": [ [ 49, 1 ] ], "l:145": [ [ 50, 1 ] ], "l:147": [ [ 51, 1 ] ], "l:149": [ [ 153, 1 ] ], "l:150": [ [ 33, 1 ] ], "l:163": [ [ 120, 1 ] ], "l:171": [ [ 70, 1 ] ], "l:172": [ [ 121, 4 ] ], "l:175": [ [ 155, 17 ] ], "l:186": [ [ 100, 1 ] ], "l:187": [ [ 101, 1 ] ], "l:189": [ [ 102, 1 ] ], "l:196": [ [ 91, 1 ] ], "l:197": [ [ 92, 1 ] ], "l:199": [ [ 93, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:201": [ [ 28, 1 ], [ 30, 3 ] ], "l:210": [ [ 103, 1 ] ], "l:211": [ [ 104, 1 ] ], "l:213": [ [ 105, 1 ] ], "l:222": [ [ 94, 1 ] ], "l:223": [ [ 95, 1 ] ], "l:225": [ [ 96, 1 ] ], "l:227": [ [ 26, 2 ], [ 125, 2 ] ], "l:242": [ [ 106, 1 ] ], "l:250": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "l:252": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "l:275": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:36": [ [ 47, 1 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:8": [ [ 1, 17 ] ], "l:84": [ [ 88, 1 ] ], "l:85": [ [ 89, 1 ] ], "l:87": [ [ 90, 1 ] ], "l:89": [ [ 35, 1 ], [ 178, 3 ] ] } }, { "key": "L6.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:110": [ [ 72, 3 ] ], "l:127": [ [ 52, 1 ] ], "l:128": [ [ 113, 4 ] ], "l:131": [ [ 19, 2 ], [ 75, 13 ], [ 111, 1 ], [ 329, 1 ] ], "l:136": [ [ 97, 1 ] ], "l:137": [ [ 98, 1 ] ], "l:139": [ [ 99, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:144": [ [ 49, 1 ] ], "l:145": [ [ 50, 1 ] ], "l:147": [ [ 51, 1 ] ], "l:148": [ [ 71, 1 ] ], "l:163": [ [ 120, 1 ] ], "l:169": [ [ 22, 1 ] ], "l:171": [ [ 70, 1 ] ], "l:172": [ [ 121, 4 ] ], "l:175": [ [ 155, 17 ] ], "l:186": [ [ 100, 1 ] ], "l:187": [ [ 101, 1 ] ], "l:189": [ [ 102, 1 ] ], "l:196": [ [ 91, 1 ] ], "l:197": [ [ 92, 1 ] ], "l:199": [ [ 93, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:201": [ [ 28, 1 ], [ 30, 3 ] ], "l:210": [ [ 103, 1 ] ], "l:211": [ [ 104, 1 ] ], "l:213": [ [ 105, 1 ] ], "l:222": [ [ 94, 1 ] ], "l:223": [ [ 95, 1 ] ], "l:225": [ [ 96, 1 ] ], "l:227": [ [ 26, 2 ], [ 125, 2 ] ], "l:242": [ [ 106, 1 ] ], "l:250": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "l:252": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "l:275": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:36": [ [ 47, 1 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:8": [ [ 1, 17 ] ], "l:84": [ [ 88, 1 ] ], "l:85": [ [ 89, 1 ] ], "l:87": [ [ 90, 1 ] ], "l:89": [ [ 35, 1 ], [ 178, 3 ] ] } }, { "key": "L7.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:108": [ [ 113, 4 ] ], "l:110": [ [ 72, 3 ] ], "l:127": [ [ 52, 1 ] ], "l:131": [ [ 19, 2 ], [ 75, 13 ], [ 111, 1 ], [ 329, 1 ] ], "l:136": [ [ 97, 1 ] ], "l:137": [ [ 98, 1 ] ], "l:139": [ [ 99, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:141": [ [ 23, 1 ], [ 66, 1 ], [ 117, 1 ], [ 330, 1 ] ], "l:142": [ [ 51, 1 ] ], "l:163": [ [ 120, 1 ] ], "l:167": [ [ 54, 3 ] ], "l:170": [ [ 53, 1 ] ], "l:171": [ [ 70, 1 ] ], "l:172": [ [ 121, 4 ] ], "l:175": [ [ 155, 17 ] ], "l:186": [ [ 100, 1 ] ], "l:187": [ [ 101, 1 ] ], "l:189": [ [ 102, 1 ] ], "l:191": [ [ 63, 2 ], [ 68, 2 ] ], "l:196": [ [ 91, 1 ] ], "l:197": [ [ 92, 1 ] ], "l:199": [ [ 93, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:201": [ [ 28, 1 ], [ 30, 3 ] ], "l:210": [ [ 103, 1 ] ], "l:211": [ [ 104, 1 ] ], "l:213": [ [ 105, 1 ] ], "l:215": [ [ 57, 2 ], [ 61, 2 ] ], "l:222": [ [ 94, 1 ] ], "l:223": [ [ 95, 1 ] ], "l:225": [ [ 96, 1 ] ], "l:227": [ [ 26, 2 ], [ 125, 2 ] ], "l:242": [ [ 106, 1 ] ], "l:250": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "l:252": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "l:275": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:36": [ [ 47, 1 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:84": [ [ 88, 1 ] ], "l:85": [ [ 89, 1 ] ], "l:87": [ [ 90, 1 ] ], "l:89": [ [ 35, 1 ], [ 178, 3 ] ] } }, { "key": "L8.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:136": [ [ 97, 1 ] ], "l:137": [ [ 98, 1 ] ], "l:139": [ [ 99, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:141": [ [ 23, 1 ], [ 66, 1 ], [ 117, 1 ], [ 330, 1 ] ], "l:161": [ [ 258, 3 ] ], "l:166": [ [ 255, 3 ] ], "l:171": [ [ 70, 1 ] ], "l:172": [ [ 121, 4 ] ], "l:175": [ [ 155, 17 ] ], "l:186": [ [ 100, 1 ] ], "l:187": [ [ 101, 1 ] ], "l:189": [ [ 102, 1 ] ], "l:191": [ [ 63, 2 ], [ 68, 2 ] ], "l:196": [ [ 91, 1 ] ], "l:197": [ [ 92, 1 ] ], "l:199": [ [ 93, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:200": [ [ 20, 1 ], [ 22, 1 ], [ 26, 3 ], [ 30, 3 ], [ 111, 1 ], [ 113, 4 ], [ 125, 2 ], [ 153, 2 ] ], "l:210": [ [ 103, 1 ] ], "l:211": [ [ 104, 1 ] ], "l:213": [ [ 105, 1 ] ], "l:215": [ [ 57, 2 ], [ 61, 2 ] ], "l:222": [ [ 94, 1 ] ], "l:223": [ [ 95, 1 ] ], "l:225": [ [ 96, 1 ] ], "l:226": [ [ 33, 3 ], [ 38, 2 ], [ 49, 2 ], [ 178, 3 ], [ 249, 6 ], [ 327, 1 ] ], "l:242": [ [ 106, 1 ] ], "l:250": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "l:252": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "l:275": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:36": [ [ 47, 1 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:84": [ [ 88, 1 ] ], "l:85": [ [ 89, 1 ] ], "l:87": [ [ 90, 1 ] ], "l:88": [ [ 19, 1 ], [ 51, 1 ], [ 71, 1 ], [ 75, 13 ], [ 329, 1 ] ] } }, { "key": "L9.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:136": [ [ 97, 1 ] ], "l:137": [ [ 98, 1 ] ], "l:139": [ [ 99, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:141": [ [ 23, 1 ], [ 66, 1 ], [ 117, 1 ], [ 330, 1 ] ], "l:159": [ [ 121, 4 ] ], "l:161": [ [ 258, 3 ] ], "l:171": [ [ 70, 1 ] ], "l:175": [ [ 155, 17 ] ], "l:186": [ [ 100, 1 ] ], "l:187": [ [ 101, 1 ] ], "l:189": [ [ 102, 1 ] ], "l:191": [ [ 63, 2 ], [ 68, 2 ] ], "l:196": [ [ 91, 1 ] ], "l:197": [ [ 92, 1 ] ], "l:199": [ [ 93, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:200": [ [ 20, 1 ], [ 22, 1 ], [ 26, 3 ], [ 30, 3 ], [ 111, 1 ], [ 113, 4 ], [ 125, 2 ], [ 153, 2 ] ], "l:210": [ [ 103, 1 ] ], "l:211": [ [ 104, 1 ] ], "l:213": [ [ 105, 1 ] ], "l:215": [ [ 57, 2 ], [ 61, 2 ] ], "l:222": [ [ 94, 1 ] ], "l:223": [ [ 95, 1 ] ], "l:225": [ [ 96, 1 ] ], "l:226": [ [ 33, 3 ], [ 38, 2 ], [ 49, 2 ], [ 178, 3 ], [ 249, 6 ], [ 327, 1 ] ], "l:242": [ [ 106, 1 ] ], "l:250": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "l:252": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "l:275": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:36": [ [ 47, 1 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:84": [ [ 88, 1 ] ], "l:85": [ [ 89, 1 ] ], "l:87": [ [ 90, 1 ] ], "l:88": [ [ 19, 1 ], [ 51, 1 ], [ 71, 1 ], [ 75, 13 ], [ 329, 1 ] ] } }, { "key": "L10.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:136": [ [ 97, 1 ] ], "l:137": [ [ 98, 1 ] ], "l:139": [ [ 99, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:140": [ [ 23, 1 ], [ 54, 5 ], [ 61, 4 ], [ 68, 2 ], [ 72, 3 ], [ 117, 1 ], [ 330, 1 ] ], "l:186": [ [ 100, 1 ] ], "l:187": [ [ 101, 1 ] ], "l:189": [ [ 102, 1 ] ], "l:190": [ [ 66, 1 ], [ 120, 5 ], [ 155, 11 ] ], "l:196": [ [ 91, 1 ] ], "l:197": [ [ 92, 1 ] ], "l:199": [ [ 93, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:200": [ [ 20, 1 ], [ 22, 1 ], [ 26, 3 ], [ 30, 3 ], [ 111, 1 ], [ 113, 4 ], [ 125, 2 ], [ 153, 2 ] ], "l:210": [ [ 103, 1 ] ], "l:211": [ [ 104, 1 ] ], "l:213": [ [ 105, 1 ] ], "l:214": [ [ 52, 2 ], [ 166, 6 ], [ 255, 3 ], [ 261, 6 ] ], "l:222": [ [ 94, 1 ] ], "l:223": [ [ 95, 1 ] ], "l:225": [ [ 96, 1 ] ], "l:226": [ [ 33, 3 ], [ 38, 2 ], [ 49, 2 ], [ 178, 3 ], [ 249, 6 ], [ 327, 1 ] ], "l:242": [ [ 106, 1 ] ], "l:250": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "l:252": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "l:274": [ [ 267, 2 ] ], "l:275": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:36": [ [ 47, 1 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:84": [ [ 88, 1 ] ], "l:85": [ [ 89, 1 ] ], "l:87": [ [ 90, 1 ] ], "l:88": [ [ 19, 1 ], [ 51, 1 ], [ 71, 1 ], [ 75, 13 ], [ 329, 1 ] ] } }, { "key": "L11.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:134": [ [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:182": [ [ 265, 2 ] ], "l:184": [ [ 30, 3 ], [ 153, 14 ] ], "l:192": [ [ 53, 2 ] ], "l:194": [ [ 76, 13 ], [ 90, 3 ], [ 329, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:206": [ [ 259, 2 ] ], "l:208": [ [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ] ], "l:216": [ [ 261, 2 ] ], "l:218": [ [ 263, 2 ] ], "l:220": [ [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ] ], "l:232": [ [ 253, 2 ] ], "l:234": [ [ 256, 2 ] ], "l:240": [ [ 51, 1 ], [ 258, 1 ] ], "l:242": [ [ 106, 1 ] ], "l:250": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "l:252": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "l:272": [ [ 269, 2 ] ], "l:275": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "l:300": [ [ 250, 1 ] ], "l:304": [ [ 251, 1 ] ], "l:308": [ [ 50, 1 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:353": [ [ 273, 17 ] ], "l:36": [ [ 47, 1 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:80": [ [ 271, 2 ] ], "l:82": [ [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ] ] } }, { "key": "L12.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:134": [ [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:180": [ [ 351, 1 ] ], "l:184": [ [ 30, 3 ], [ 153, 14 ] ], "l:194": [ [ 76, 13 ], [ 90, 3 ], [ 329, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:204": [ [ 350, 1 ] ], "l:208": [ [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ] ], "l:216": [ [ 261, 2 ] ], "l:220": [ [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ] ], "l:230": [ [ 349, 1 ] ], "l:238": [ [ 267, 1 ] ], "l:242": [ [ 106, 1 ] ], "l:250": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "l:252": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "l:270": [ [ 298, 2 ] ], "l:275": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "l:299": [ [ 290, 1 ] ], "l:302": [ [ 291, 1 ] ], "l:303": [ [ 292, 1 ] ], "l:306": [ [ 60, 1 ] ], "l:307": [ [ 59, 1 ] ], "l:310": [ [ 297, 1 ] ], "l:337": [ [ 300, 17 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:353": [ [ 273, 17 ] ], "l:359": [ [ 65, 1 ], [ 67, 1 ], [ 317, 2 ], [ 320, 7 ], [ 328, 1 ], [ 344, 5 ] ], "l:36": [ [ 47, 1 ] ], "l:360": [ [ 268, 1 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:79": [ [ 352, 1 ] ], "l:82": [ [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ] ] } }, { "key": "L13.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:134": [ [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:180": [ [ 351, 1 ] ], "l:184": [ [ 30, 3 ], [ 153, 14 ] ], "l:194": [ [ 76, 13 ], [ 90, 3 ], [ 329, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:204": [ [ 350, 1 ] ], "l:208": [ [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ] ], "l:220": [ [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ] ], "l:230": [ [ 349, 1 ] ], "l:236": [ [ 50, 1 ] ], "l:237": [ [ 267, 1 ] ], "l:242": [ [ 106, 1 ] ], "l:250": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "l:252": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "l:267": [ [ 403, 17 ] ], "l:268": [ [ 253, 2 ] ], "l:275": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "l:289": [ [ 51, 1 ], [ 53, 2 ], [ 256, 11 ], [ 269, 3 ] ], "l:296": [ [ 251, 1 ] ], "l:297": [ [ 250, 1 ] ], "l:335": [ [ 369, 17 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:343": [ [ 272, 1 ], [ 353, 16 ] ], "l:353": [ [ 273, 17 ] ], "l:357": [ [ 386, 17 ] ], "l:36": [ [ 47, 1 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:79": [ [ 352, 1 ] ], "l:82": [ [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ] ] } }, { "key": "L14.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:180": [ [ 351, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:204": [ [ 350, 1 ] ], "l:228": [ [ 323, 1 ] ], "l:229": [ [ 349, 1 ] ], "l:236": [ [ 50, 1 ] ], "l:250": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "l:252": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "l:265": [ [ 479, 17 ] ], "l:275": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "l:287": [ [ 513, 17 ] ], "l:294": [ [ 445, 17 ] ], "l:311": [ [ 462, 17 ] ], "l:315": [ [ 65, 1 ], [ 324, 3 ], [ 344, 5 ], [ 420, 8 ] ], "l:317": [ [ 428, 17 ] ], "l:321": [ [ 59, 2 ], [ 67, 1 ], [ 106, 1 ], [ 267, 1 ], [ 290, 3 ], [ 297, 9 ] ], "l:323": [ [ 306, 13 ], [ 320, 3 ], [ 328, 1 ] ], "l:325": [ [ 531, 2 ] ], "l:328": [ [ 496, 17 ] ], "l:335": [ [ 369, 17 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:343": [ [ 272, 1 ], [ 353, 16 ] ], "l:347": [ [ 530, 1 ] ], "l:353": [ [ 273, 17 ] ], "l:357": [ [ 386, 17 ] ], "l:36": [ [ 47, 1 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:79": [ [ 352, 1 ] ] } }, { "key": "L15.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:180": [ [ 351, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:202": [ [ 22, 1 ] ], "l:203": [ [ 350, 1 ] ], "l:228": [ [ 323, 1 ] ], "l:250": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "l:252": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "l:263": [ [ 28, 1 ], [ 30, 3 ], [ 153, 13 ] ], "l:275": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "l:279": [ [ 23, 1 ], [ 26, 2 ], [ 66, 1 ], [ 111, 1 ], [ 113, 3 ], [ 117, 1 ], [ 120, 7 ], [ 330, 1 ] ], "l:287": [ [ 513, 17 ] ], "l:294": [ [ 445, 17 ] ], "l:311": [ [ 462, 17 ] ], "l:315": [ [ 65, 1 ], [ 324, 3 ], [ 344, 5 ], [ 420, 8 ] ], "l:317": [ [ 428, 17 ] ], "l:319": [ [ 21, 1 ], [ 116, 1 ] ], "l:326": [ [ 33, 3 ], [ 38, 2 ], [ 50, 1 ], [ 166, 1 ], [ 168, 4 ], [ 178, 3 ], [ 251, 1 ], [ 327, 1 ], [ 404, 1 ] ], "l:335": [ [ 369, 17 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:343": [ [ 272, 1 ], [ 353, 16 ] ], "l:345": [ [ 264, 1 ] ], "l:353": [ [ 273, 17 ] ], "l:356": [ [ 49, 1 ], [ 51, 2 ], [ 249, 2 ], [ 252, 12 ] ], "l:357": [ [ 386, 17 ] ], "l:36": [ [ 47, 1 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:79": [ [ 352, 1 ] ] } }, { "key": "L16.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:178": [ [ 67, 1 ] ], "l:179": [ [ 351, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:202": [ [ 22, 1 ] ], "l:250": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "l:252": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "l:256": [ [ 53, 8 ], [ 265, 2 ], [ 269, 3 ], [ 290, 3 ], [ 297, 1 ] ], "l:263": [ [ 28, 1 ], [ 30, 3 ], [ 153, 13 ] ], "l:275": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "l:277": [ [ 63, 2 ], [ 70, 6 ], [ 315, 2 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 349, 2 ] ], "l:287": [ [ 513, 17 ] ], "l:294": [ [ 445, 17 ] ], "l:311": [ [ 462, 17 ] ], "l:313": [ [ 106, 1 ], [ 267, 1 ] ], "l:326": [ [ 33, 3 ], [ 38, 2 ], [ 50, 1 ], [ 166, 1 ], [ 168, 4 ], [ 178, 3 ], [ 251, 1 ], [ 327, 1 ], [ 404, 1 ] ], "l:335": [ [ 369, 17 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:340": [ [ 61, 2 ], [ 298, 9 ], [ 308, 4 ], [ 313, 2 ] ], "l:341": [ [ 68, 1 ], [ 76, 7 ], [ 403, 1 ], [ 405, 8 ] ], "l:343": [ [ 272, 1 ], [ 353, 16 ] ], "l:345": [ [ 264, 1 ] ], "l:351": [ [ 83, 6 ], [ 90, 5 ], [ 329, 1 ], [ 414, 1 ], [ 416, 4 ] ], "l:354": [ [ 19, 1 ], [ 95, 1 ], [ 97, 1 ], [ 479, 10 ], [ 490, 4 ] ], "l:357": [ [ 386, 17 ] ], "l:36": [ [ 47, 1 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:79": [ [ 352, 1 ] ] } }, { "key": "L17.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:178": [ [ 67, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:250": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "l:252": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "l:254": [ [ 65, 1 ], [ 285, 5 ], [ 324, 3 ], [ 346, 3 ], [ 351, 1 ], [ 353, 4 ] ], "l:260": [ [ 425, 1 ] ], "l:261": [ [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "l:287": [ [ 513, 17 ] ], "l:292": [ [ 26, 2 ], [ 47, 1 ], [ 49, 1 ], [ 51, 2 ], [ 66, 1 ], [ 249, 2 ], [ 252, 8 ] ], "l:326": [ [ 33, 3 ], [ 38, 2 ], [ 50, 1 ], [ 166, 1 ], [ 168, 4 ], [ 178, 3 ], [ 251, 1 ], [ 327, 1 ], [ 404, 1 ] ], "l:335": [ [ 369, 17 ] ], "l:338": [ [ 357, 12 ], [ 420, 5 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:341": [ [ 68, 1 ], [ 76, 7 ], [ 403, 1 ], [ 405, 8 ] ], "l:345": [ [ 264, 1 ] ], "l:349": [ [ 426, 17 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:62": [ [ 48, 1 ] ], "l:71": [ [ 18, 1 ] ], "l:77": [ [ 260, 4 ], [ 272, 13 ] ], "l:78": [ [ 352, 1 ] ] } }, { "key": "L18.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:247": [ [ 56, 1 ] ], "l:248": [ [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ] ], "l:259": [ [ 286, 1 ] ], "l:261": [ [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "l:287": [ [ 513, 17 ] ], "l:290": [ [ 53, 3 ], [ 67, 1 ], [ 245, 4 ], [ 265, 3 ], [ 269, 3 ], [ 287, 1 ], [ 336, 1 ], [ 391, 1 ] ], "l:326": [ [ 33, 3 ], [ 38, 2 ], [ 50, 1 ], [ 166, 1 ], [ 168, 4 ], [ 178, 3 ], [ 251, 1 ], [ 327, 1 ], [ 404, 1 ] ], "l:333": [ [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ] ], "l:34": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "l:345": [ [ 264, 1 ] ], "l:349": [ [ 426, 17 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:451": [ [ 57, 4 ], [ 285, 1 ], [ 288, 5 ], [ 296, 5 ], [ 337, 1 ], [ 393, 1 ] ], "l:60": [ [ 190, 17 ] ], "l:69": [ [ 42, 3 ], [ 213, 1 ], [ 215, 2 ], [ 218, 4 ], [ 227, 1 ], [ 241, 4 ], [ 334, 2 ] ] } }, { "key": "L19.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:246": [ [ 204, 1 ] ], "l:248": [ [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ] ], "l:259": [ [ 286, 1 ] ], "l:261": [ [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "l:284": [ [ 202, 1 ] ], "l:285": [ [ 37, 1 ], [ 184, 3 ], [ 188, 13 ] ], "l:31": [ [ 201, 1 ] ], "l:32": [ [ 18, 1 ], [ 26, 2 ], [ 36, 1 ], [ 66, 1 ], [ 76, 2 ], [ 172, 6 ], [ 181, 3 ], [ 187, 1 ] ], "l:332": [ [ 203, 1 ] ], "l:333": [ [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ] ], "l:344": [ [ 205, 1 ] ], "l:349": [ [ 426, 17 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:451": [ [ 57, 4 ], [ 285, 1 ], [ 288, 5 ], [ 296, 5 ], [ 337, 1 ], [ 393, 1 ] ] } }, { "key": "L20.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:243": [ [ 54, 1 ] ], "l:245": [ [ 50, 1 ] ], "l:248": [ [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ] ], "l:257": [ [ 55, 1 ] ], "l:261": [ [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "l:281": [ [ 166, 1 ] ], "l:283": [ [ 53, 1 ] ], "l:285": [ [ 37, 1 ], [ 184, 3 ], [ 188, 13 ] ], "l:30": [ [ 56, 1 ] ], "l:32": [ [ 18, 1 ], [ 26, 2 ], [ 36, 1 ], [ 66, 1 ], [ 76, 2 ], [ 172, 6 ], [ 181, 3 ], [ 187, 1 ] ], "l:330": [ [ 33, 1 ] ], "l:333": [ [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ] ], "l:344": [ [ 205, 1 ] ], "l:349": [ [ 426, 17 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:451": [ [ 57, 4 ], [ 285, 1 ], [ 288, 5 ], [ 296, 5 ], [ 337, 1 ], [ 393, 1 ] ] } }, { "key": "L21.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:243": [ [ 54, 1 ] ], "l:248": [ [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ] ], "l:257": [ [ 55, 1 ] ], "l:261": [ [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "l:280": [ [ 168, 1 ] ], "l:285": [ [ 37, 1 ], [ 184, 3 ], [ 188, 13 ] ], "l:30": [ [ 56, 1 ] ], "l:32": [ [ 18, 1 ], [ 26, 2 ], [ 36, 1 ], [ 66, 1 ], [ 76, 2 ], [ 172, 6 ], [ 181, 3 ], [ 187, 1 ] ], "l:329": [ [ 169, 1 ] ], "l:333": [ [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ] ], "l:344": [ [ 205, 1 ] ], "l:349": [ [ 426, 17 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:451": [ [ 57, 4 ], [ 285, 1 ], [ 288, 5 ], [ 296, 5 ], [ 337, 1 ], [ 393, 1 ] ] } }, { "key": "L22.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:248": [ [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ] ], "l:261": [ [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "l:27": [ [ 50, 1 ] ], "l:285": [ [ 37, 1 ], [ 184, 3 ], [ 188, 13 ] ], "l:32": [ [ 18, 1 ], [ 26, 2 ], [ 36, 1 ], [ 66, 1 ], [ 76, 2 ], [ 172, 6 ], [ 181, 3 ], [ 187, 1 ] ], "l:333": [ [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ] ], "l:344": [ [ 205, 1 ] ], "l:349": [ [ 426, 17 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:451": [ [ 57, 4 ], [ 285, 1 ], [ 288, 5 ], [ 296, 5 ], [ 337, 1 ], [ 393, 1 ] ] } }, { "key": "L23.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:248": [ [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ] ], "l:26": [ [ 33, 3 ], [ 53, 4 ], [ 166, 1 ], [ 168, 4 ] ], "l:261": [ [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "l:27": [ [ 50, 1 ] ], "l:285": [ [ 37, 1 ], [ 184, 3 ], [ 188, 13 ] ], "l:32": [ [ 18, 1 ], [ 26, 2 ], [ 36, 1 ], [ 66, 1 ], [ 76, 2 ], [ 172, 6 ], [ 181, 3 ], [ 187, 1 ] ], "l:333": [ [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ] ], "l:349": [ [ 426, 17 ] ], "l:361": [ [ 38, 2 ], [ 42, 1 ], [ 178, 3 ], [ 201, 6 ], [ 213, 1 ], [ 215, 1 ], [ 327, 1 ], [ 334, 1 ], [ 404, 1 ] ], "l:364": [ [ 340, 1 ] ], "l:372": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "l:385": [ [ 293, 1 ] ], "l:393": [ [ 136, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:413": [ [ 294, 1 ] ], "l:423": [ [ 342, 1 ] ], "l:433": [ [ 295, 1 ] ], "l:443": [ [ 343, 1 ] ], "l:451": [ [ 57, 4 ], [ 285, 1 ], [ 288, 5 ], [ 296, 5 ], [ 337, 1 ], [ 393, 1 ] ] } }, { "key": "L24.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 339, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:24": [ [ 43, 10 ], [ 61, 5 ], [ 67, 2 ], [ 70, 6 ], [ 78, 11 ], [ 216, 1 ], [ 218, 4 ], [ 224, 8 ], [ 233, 4 ], [ 238, 4 ], [ 243, 4 ], [ 248, 3 ], [ 252, 10 ], [ 263, 1 ], [ 265, 3 ], [ 269, 16 ], [ 286, 2 ], [ 301, 6 ], [ 308, 4 ], [ 313, 4 ], [ 318, 1 ], [ 320, 3 ], [ 324, 3 ], [ 328, 2 ], [ 332, 1 ], [ 335, 2 ], [ 338, 1 ], [ 346, 12 ], [ 359, 4 ], [ 364, 3 ], [ 369, 9 ], [ 379, 13 ], [ 394, 9 ], [ 405, 7 ], [ 414, 1 ], [ 416, 10 ], [ 443, 15 ] ], "l:364": [ [ 340, 1 ] ], "l:369": [ [ 20, 1 ], [ 69, 1 ], [ 97, 4 ], [ 102, 4 ], [ 344, 2 ], [ 458, 6 ], [ 465, 14 ], [ 480, 9 ], [ 490, 4 ], [ 495, 4 ], [ 500, 4 ], [ 505, 1 ], [ 507, 6 ], [ 515, 4 ], [ 520, 1 ], [ 530, 1 ], [ 533, 138 ] ], "l:385": [ [ 293, 1 ] ], "l:390": [ [ 671, 204 ] ], "l:403": [ [ 341, 1 ] ], "l:408": [ [ 875, 204 ] ], "l:413": [ [ 294, 1 ] ], "l:418": [ [ 1079, 204 ] ], "l:423": [ [ 342, 1 ] ], "l:428": [ [ 1283, 204 ] ], "l:433": [ [ 295, 1 ] ], "l:438": [ [ 1487, 204 ] ], "l:443": [ [ 343, 1 ] ], "l:448": [ [ 1691, 204 ] ], "l:451": [ [ 57, 4 ], [ 285, 1 ], [ 288, 5 ], [ 296, 5 ], [ 337, 1 ], [ 393, 1 ] ] } }, { "key": "L25.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:12": [ [ 40, 2 ], [ 197, 14 ], [ 404, 1 ] ], "l:2": [ [ 29, 1 ] ], "l:23": [ [ 426, 17 ] ], "l:362": [ [ 42, 1 ], [ 53, 4 ], [ 76, 2 ], [ 211, 1 ], [ 213, 3 ], [ 334, 1 ], [ 1895, 5 ] ], "l:368": [ [ 18, 2 ], [ 21, 2 ], [ 90, 6 ], [ 106, 3 ], [ 110, 4 ] ], "l:383": [ [ 1900, 17 ] ], "l:389": [ [ 23, 3 ], [ 114, 4 ], [ 120, 9 ], [ 330, 1 ] ], "l:401": [ [ 1917, 17 ] ], "l:403": [ [ 341, 1 ] ], "l:407": [ [ 26, 3 ], [ 66, 1 ], [ 129, 12 ], [ 331, 1 ] ], "l:411": [ [ 1934, 17 ] ], "l:417": [ [ 30, 2 ], [ 141, 15 ] ], "l:421": [ [ 1951, 17 ] ], "l:427": [ [ 32, 2 ], [ 156, 11 ], [ 168, 4 ] ], "l:431": [ [ 1968, 17 ] ], "l:437": [ [ 34, 3 ], [ 172, 13 ], [ 392, 1 ] ], "l:441": [ [ 1985, 17 ] ], "l:447": [ [ 37, 3 ], [ 185, 12 ], [ 327, 1 ], [ 333, 1 ] ], "l:451": [ [ 57, 4 ], [ 285, 1 ], [ 288, 5 ], [ 296, 5 ], [ 337, 1 ], [ 393, 1 ] ] } }, { "key": "L26.post", "nodes": { "l:0": [ [ 1, 17 ] ], "l:10": [ [ 30, 1 ], [ 141, 1 ], [ 145, 1 ], [ 1631, 3 ], [ 1646, 3 ], [ 1650, 7 ], [ 1694, 1 ] ] } }, { "key": "output", "nodes": { "l:0": [ [ 1, 17 ] ] } } ], "width": 2048 }, "runtime_contract": { "causal_attention": true, "dtype": "float32", "generation": { "do_sample": false, "greedy": true }, "recommended_attention_implementation_for_analysis": "eager", "unsupported_inferences": [ "Static graph ancestry is not dynamic causal relevance.", "Unused-on-one-input activations require an input-specific trace.", "Other numerical precisions are outside the exactness contract." ] }, "schedule": { "assignment": [ { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": null, "node": "l:8", "routing": null }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:46", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:48", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:50", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:45", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:47", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:49", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "mlp", "layer": 2, "node": "l:43", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:7", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:54", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:55", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:57", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:52", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "mlp", "layer": 1, "node": "l:51", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:41", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:38", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:39", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:58", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 4, "node": "l:36", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:64", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:65", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:67", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 4, "node": "l:62", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:73", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:74", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:76", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 4, "node": "l:71", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:105", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:118", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:119", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:121", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:116", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 0, "node": "l:114", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:115", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 5, "node": "l:112", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 1, "node": "l:165", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "mlp", "layer": 4, "node": "l:163", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:102", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:103", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:106", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:100", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:107", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 0, "node": "l:99", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:98", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "mlp", "layer": 2, "node": "l:97", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:96", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:125", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:126", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:123", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 5, "node": "l:122", "routing": "attn" }, { "cancel_layer": 8, "cancel_sublayer": "mlp", "layer": 6, "node": "l:110", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "mlp", "layer": 0, "node": "l:127", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:130", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 1, "node": "l:128", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:108", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 0, "node": "l:133", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 6, "node": "l:131", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:154", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:155", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:157", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:152", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:158", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:151", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:150", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 5, "node": "l:149", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:148", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:169", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "mlp", "layer": 7, "node": "l:170", "routing": "attn" }, { "cancel_layer": 8, "cancel_sublayer": "mlp", "layer": 7, "node": "l:167", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "mlp", "layer": 8, "node": "l:166", "routing": "attn" }, { "cancel_layer": 10, "cancel_sublayer": "mlp", "layer": 8, "node": "l:161", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "mlp", "layer": 0, "node": "l:171", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 0, "node": "l:174", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 4, "node": "l:172", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 9, "node": "l:159", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 0, "node": "l:177", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 5, "node": "l:175", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:92", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:93", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:95", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:90", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 0, "node": "l:144", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 0, "node": "l:145", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 0, "node": "l:147", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:142", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 8, "node": "l:88", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:84", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:85", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:87", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 11, "node": "l:82", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 8, "node": "l:200", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:196", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:197", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:199", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 11, "node": "l:194", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 8, "node": "l:226", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:222", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:223", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:225", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 11, "node": "l:220", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:140", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:136", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:137", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:139", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 11, "node": "l:134", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:190", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:186", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:187", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:189", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 11, "node": "l:184", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:214", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:210", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:211", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:213", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 11, "node": "l:208", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:267", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "mlp", "layer": 10, "node": "l:274", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 11, "node": "l:353", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:272", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:337", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:270", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:289", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:268", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:265", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:59", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:376", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:300", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:304", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:308", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:299", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:302", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:303", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:306", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:307", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:310", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:297", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:296", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 14, "node": "l:294", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 14, "node": "l:315", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:321", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 14, "node": "l:311", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 14, "node": "l:317", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:323", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:325", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 15, "node": "l:356", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 15, "node": "l:319", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 16, "node": "l:340", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 16, "node": "l:313", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 17, "node": "l:292", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:68", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:379", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:80", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:182", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:206", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:192", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:216", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:232", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:218", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:234", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:240", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:360", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:238", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:237", "routing": "attn" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:236", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 12, "node": "l:230", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 14, "node": "l:229", "routing": "attn" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 14, "node": "l:228", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 12, "node": "l:204", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:203", "routing": "attn" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 15, "node": "l:202", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 12, "node": "l:180", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:179", "routing": "attn" }, { "cancel_layer": 18, "cancel_sublayer": "attn", "layer": 16, "node": "l:178", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "attn", "layer": 12, "node": "l:79", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "attn", "layer": 17, "node": "l:78", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:382", "routing": "mlp" }, { "cancel_layer": 19, "cancel_sublayer": "attn", "layer": 5, "node": "l:34", "routing": "mlp" }, { "cancel_layer": 19, "cancel_sublayer": "attn", "layer": 5, "node": "l:60", "routing": "mlp" }, { "cancel_layer": 19, "cancel_sublayer": "attn", "layer": 18, "node": "l:69", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "mlp", "layer": 19, "node": "l:32", "routing": "attn" }, { "cancel_layer": 18, "cancel_sublayer": "attn", "layer": 5, "node": "l:250", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 5, "node": "l:252", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "attn", "layer": 17, "node": "l:254", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "mlp", "layer": 18, "node": "l:248", "routing": "attn" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 15, "node": "l:263", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 5, "node": "l:275", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 16, "node": "l:277", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "mlp", "layer": 17, "node": "l:261", "routing": "attn" }, { "cancel_layer": 19, "cancel_sublayer": "attn", "layer": 14, "node": "l:287", "routing": "mlp" }, { "cancel_layer": 19, "cancel_sublayer": "mlp", "layer": 18, "node": "l:290", "routing": "mlp" }, { "cancel_layer": 19, "cancel_sublayer": "mlp", "layer": 15, "node": "l:326", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "mlp", "layer": 19, "node": "l:285", "routing": "attn" }, { "cancel_layer": 18, "cancel_sublayer": "attn", "layer": 13, "node": "l:335", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "attn", "layer": 17, "node": "l:338", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 16, "node": "l:341", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "mlp", "layer": 18, "node": "l:333", "routing": "attn" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 16, "node": "l:351", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 16, "node": "l:354", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 13, "node": "l:357", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "mlp", "layer": 17, "node": "l:349", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:374", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:377", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:380", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "mlp", "layer": 5, "node": "l:372", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:395", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:397", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:399", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "mlp", "layer": 5, "node": "l:393", "routing": "attn" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:347", "routing": "mlp" }, { "cancel_layer": 19, "cancel_sublayer": "mlp", "layer": 15, "node": "l:345", "routing": "mlp" }, { "cancel_layer": 20, "cancel_sublayer": "mlp", "layer": 19, "node": "l:31", "routing": "mlp" }, { "cancel_layer": 21, "cancel_sublayer": "mlp", "layer": 20, "node": "l:245", "routing": "attn" }, { "cancel_layer": 19, "cancel_sublayer": "mlp", "layer": 18, "node": "l:247", "routing": "mlp" }, { "cancel_layer": 20, "cancel_sublayer": "mlp", "layer": 19, "node": "l:246", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 17, "node": "l:260", "routing": "mlp" }, { "cancel_layer": 20, "cancel_sublayer": "mlp", "layer": 18, "node": "l:259", "routing": "mlp" }, { "cancel_layer": 20, "cancel_sublayer": "mlp", "layer": 19, "node": "l:284", "routing": "mlp" }, { "cancel_layer": 21, "cancel_sublayer": "mlp", "layer": 20, "node": "l:283", "routing": "attn" }, { "cancel_layer": 20, "cancel_sublayer": "mlp", "layer": 19, "node": "l:332", "routing": "mlp" }, { "cancel_layer": 22, "cancel_sublayer": "attn", "layer": 20, "node": "l:243", "routing": "mlp" }, { "cancel_layer": 22, "cancel_sublayer": "attn", "layer": 20, "node": "l:257", "routing": "mlp" }, { "cancel_layer": 21, "cancel_sublayer": "mlp", "layer": 20, "node": "l:281", "routing": "mlp" }, { "cancel_layer": 22, "cancel_sublayer": "attn", "layer": 21, "node": "l:280", "routing": "mlp" }, { "cancel_layer": 21, "cancel_sublayer": "mlp", "layer": 20, "node": "l:330", "routing": "mlp" }, { "cancel_layer": 22, "cancel_sublayer": "mlp", "layer": 21, "node": "l:329", "routing": "mlp" }, { "cancel_layer": 23, "cancel_sublayer": "attn", "layer": 22, "node": "l:28", "routing": "attn" }, { "cancel_layer": 23, "cancel_sublayer": "attn", "layer": 19, "node": "l:344", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "attn", "layer": 22, "node": "l:27", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "mlp", "layer": 23, "node": "l:361", "routing": "mlp" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 24, "node": "l:24", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:23", "routing": "attn" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 24, "node": "l:369", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:368", "routing": "attn" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 24, "node": "l:390", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:389", "routing": "attn" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 24, "node": "l:408", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:407", "routing": "attn" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 24, "node": "l:418", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "attn", "layer": 25, "node": "l:417", "routing": "attn" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 24, "node": "l:428", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "attn", "layer": 25, "node": "l:427", "routing": "attn" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 24, "node": "l:438", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:437", "routing": "attn" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 24, "node": "l:448", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "attn", "layer": 25, "node": "l:447", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:4", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:5", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:9", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 1, "node": "l:2", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:21", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:20", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:19", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "mlp", "layer": 2, "node": "l:18", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:17", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:16", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:22", "routing": "mlp" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 5, "node": "l:14", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:12", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:366", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:367", "routing": "mlp" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 5, "node": "l:364", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "attn", "layer": 25, "node": "l:362", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:387", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:388", "routing": "mlp" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 4, "node": "l:385", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:383", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:405", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:406", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "attn", "layer": 5, "node": "l:403", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:401", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:415", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:416", "routing": "mlp" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 4, "node": "l:413", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:411", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:425", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:426", "routing": "mlp" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 5, "node": "l:423", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:421", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:435", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:436", "routing": "mlp" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 4, "node": "l:433", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:431", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:445", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:446", "routing": "mlp" }, { "cancel_layer": 25, "cancel_sublayer": "mlp", "layer": 5, "node": "l:443", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:441", "routing": "mlp" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 26, "node": "l:10", "routing": "attn" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 18, "node": "l:451", "routing": "mlp" }, { "cancel_layer": 36, "cancel_sublayer": "attn", "layer": 26, "node": "l:0", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 7, "node": "l:215", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 7, "node": "l:191", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 7, "node": "l:141", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 4, "node": "l:227", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 4, "node": "l:201", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 4, "node": "l:89", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 0, "node": "l:242", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:359", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 13, "node": "l:343", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:328", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 15, "node": "l:279", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 16, "node": "l:256", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 17, "node": "l:77", "routing": "mlp" }, { "cancel_layer": 22, "cancel_sublayer": "attn", "layer": 20, "node": "l:30", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "mlp", "layer": 23, "node": "l:26", "routing": "mlp" } ], "layers": [ { "active_attention_heads": 1, "active_mlp_neurons": 16, "attention_operations": [], "emitted_attention_heads": 0, "index": 0, "mlp_operations": [ { "hidden_slots": [ [ 1, 1 ] ], "id": "L0.mlp.0", "node": "l:46", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 17 ] ], "source_columns_b": null, "target_columns": [ [ 18, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 2, 1 ] ], "id": "L0.mlp.1", "node": "l:48", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 17 ] ], "source_columns_b": null, "target_columns": [ [ 19, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 1 ] ], "id": "L0.mlp.2", "node": "l:50", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 17 ] ], "source_columns_b": null, "target_columns": [ [ 20, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 4, 1 ] ], "id": "L0.mlp.3", "node": "l:7", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 17 ] ], "source_columns_b": null, "target_columns": [ [ 21, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 1 ] ], "id": "L0.mlp.4", "node": "l:105", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 17 ] ], "source_columns_b": null, "target_columns": [ [ 22, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 6, 10 ] ], "id": "L0.mlp.5", "node": "l:114", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 17 ] ], "source_columns_b": null, "target_columns": [ [ 23, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L0.mlp.6", "node": "l:54", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 24, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.7", "node": "l:55", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 25, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.8", "node": "l:57", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 26, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.9", "node": "l:102", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 27, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.10", "node": "l:103", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 28, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.11", "node": "l:106", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 29, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.12", "node": "l:154", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 30, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.13", "node": "l:155", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 31, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.14", "node": "l:157", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 32, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.15", "node": "l:99", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 33, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.16", "node": "l:151", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.17", "node": "l:118", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 35, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.18", "node": "l:119", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 36, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.19", "node": "l:121", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 37, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.20", "node": "l:130", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 38, 4 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.21", "node": "l:174", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 42, 4 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.22", "node": "l:92", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 46, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.23", "node": "l:93", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 47, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.24", "node": "l:95", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 48, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.25", "node": "l:144", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 49, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.26", "node": "l:145", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.27", "node": "l:147", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 51, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.28", "node": "l:127", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 52, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.29", "node": "l:133", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 53, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.30", "node": "l:171", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 70, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.31", "node": "l:177", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 71, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.32", "node": "l:84", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 88, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.33", "node": "l:85", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 89, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.34", "node": "l:87", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 90, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.35", "node": "l:196", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 91, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.36", "node": "l:197", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 92, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.37", "node": "l:199", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 93, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.38", "node": "l:222", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 94, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.39", "node": "l:223", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 95, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.40", "node": "l:225", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 96, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.41", "node": "l:136", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 97, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.42", "node": "l:137", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 98, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.43", "node": "l:139", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 99, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.44", "node": "l:186", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 100, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.45", "node": "l:187", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 101, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.46", "node": "l:189", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 102, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.47", "node": "l:210", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 103, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.48", "node": "l:211", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 104, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.49", "node": "l:213", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 105, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.50", "node": "l:242", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 106, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.51", "node": "l:4", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 107, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.52", "node": "l:5", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 108, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.53", "node": "l:9", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 109, 1 ] ], "type": "compute_literal_value" } ], "newly_computed": [ "l:46", "l:48", "l:50", "l:7", "l:54", "l:55", "l:57", "l:53", "l:56", "l:105", "l:118", "l:119", "l:121", "l:117", "l:114", "l:102", "l:103", "l:106", "l:101", "l:104", "l:99", "l:127", "l:130", "l:129", "l:133", "l:154", "l:155", "l:157", "l:153", "l:151", "l:171", "l:174", "l:177", "l:92", "l:93", "l:95", "l:91", "l:144", "l:145", "l:147", "l:143", "l:146", "l:84", "l:85", "l:87", "l:83", "l:86", "l:196", "l:197", "l:199", "l:195", "l:198", "l:222", "l:223", "l:225", "l:221", "l:224", "l:136", "l:137", "l:139", "l:135", "l:138", "l:186", "l:187", "l:189", "l:185", "l:188", "l:210", "l:211", "l:213", "l:209", "l:212", "l:4", "l:5", "l:9", "l:3", "l:6", "l:242" ] }, { "active_attention_heads": 4, "active_mlp_neurons": 35, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L1.attn.0", "key_source_columns": [ [ 21, 1 ], [ 26, 1 ] ], "node": "l:52", "query_source_columns": [ [ 24, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 21, 1 ] ], "source_columns_b": null, "target_columns": [ [ 27, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L1.attn.1", "key_source_columns": [ [ 22, 1 ], [ 29, 1 ] ], "node": "l:100", "query_source_columns": [ [ 27, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 22, 1 ] ], "source_columns_b": null, "target_columns": [ [ 28, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L1.attn.2", "key_source_columns": [ [ 21, 1 ], [ 109, 1 ] ], "node": "l:2", "query_source_columns": [ [ 107, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 21, 1 ] ], "source_columns_b": null, "target_columns": [ [ 29, 1 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L1.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 27, 3 ], [ 107, 3 ] ], "type": "cancel" } ], "emitted_attention_heads": 4, "index": 1, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L1.mlp.0", "node": "l:45", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ] ], "source_columns_b": null, "target_columns": [ [ 108, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L1.mlp.1", "node": "l:47", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 1 ] ], "source_columns_b": null, "target_columns": [ [ 107, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 2 ] ], "id": "L1.mlp.2", "node": "l:49", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 20, 1 ] ], "source_columns_b": null, "target_columns": [ [ 110, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 7, 2 ] ], "id": "L1.mlp.3", "node": "l:51", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 27, 1 ] ], "source_columns_b": null, "target_columns": [ [ 111, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 9, 2 ] ], "id": "L1.mlp.4", "node": "l:165", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 27, 1 ] ], "source_columns_b": null, "target_columns": [ [ 112, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 11, 4 ] ], "id": "L1.mlp.5", "node": "l:128", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 22, 1 ], [ 38, 4 ] ], "source_columns_b": null, "target_columns": [ [ 113, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 15, 2 ] ], "id": "L1.mlp.6", "node": "l:107", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 22, 1 ] ], "source_columns_b": null, "target_columns": [ [ 117, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 17, 2 ] ], "id": "L1.mlp.7", "node": "l:21", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 21, 1 ] ], "source_columns_b": null, "target_columns": [ [ 118, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L1.mlp.8", "node": "l:20", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 119, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 19, 8 ] ], "id": "L1.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 38, 4 ] ], "source_columns_b": null, "target_columns": [ [ 38, 4 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 27, 2 ] ], "id": "L1.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 24, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 29, 2 ] ], "id": "L1.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 25, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 31, 2 ] ], "id": "L1.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 26, 1 ] ], "source_columns_b": null, "target_columns": [ [ 26, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 33, 2 ] ], "id": "L1.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 22, 1 ] ], "source_columns_b": null, "target_columns": [ [ 22, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:45", "l:47", "l:49", "l:44", "l:52", "l:51", "l:165", "l:100", "l:107", "l:128", "l:2", "l:21", "l:20" ] }, { "active_attention_heads": 3, "active_mlp_neurons": 73, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L2.attn.0", "key_source_columns": [ [ 28, 1 ] ], "node": "l:98", "query_source_columns": [ [ 33, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 117, 1 ] ], "source_columns_b": null, "target_columns": [ [ 24, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L2.attn.1", "key_source_columns": [ [ 29, 1 ] ], "node": "l:19", "query_source_columns": [ [ 119, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 118, 1 ] ], "source_columns_b": null, "target_columns": [ [ 25, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L2.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 27, 2 ], [ 33, 1 ], [ 117, 3 ] ], "type": "cancel" } ], "emitted_attention_heads": 3, "index": 2, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L2.mlp.0", "node": "l:43", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 108, 1 ], [ 107, 1 ], [ 110, 1 ] ], "source_columns_b": null, "target_columns": [ [ 26, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 32 ] ], "id": "L2.mlp.1", "node": "l:97", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 27, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 35, 32 ] ], "id": "L2.mlp.2", "node": "l:18", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 28, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 67, 2 ] ], "id": "L2.mlp.3", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 108, 1 ] ], "source_columns_b": null, "target_columns": [ [ 108, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 69, 2 ] ], "id": "L2.mlp.4", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 107, 1 ] ], "source_columns_b": null, "target_columns": [ [ 107, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 71, 2 ] ], "id": "L2.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 110, 1 ] ], "source_columns_b": null, "target_columns": [ [ 110, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:43", "l:42", "l:98", "l:97", "l:19", "l:18" ] }, { "active_attention_heads": 3, "active_mlp_neurons": 25, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L3.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:96", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 27, 1 ] ], "source_columns_b": null, "target_columns": [ [ 24, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L3.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:17", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 28, 1 ] ], "source_columns_b": null, "target_columns": [ [ 25, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 1 ], "id": "L3.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 24, 2 ] ], "type": "cancel" } ], "emitted_attention_heads": 3, "index": 3, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L3.mlp.0", "node": "l:41", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 26, 1 ], [ 111, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L3.mlp.1", "node": "l:387", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 38, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 2 ] ], "id": "L3.mlp.2", "node": "l:388", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 39, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 7, 2 ] ], "id": "L3.mlp.3", "node": "l:415", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 40, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 9, 2 ] ], "id": "L3.mlp.4", "node": "l:416", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 41, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 11, 2 ] ], "id": "L3.mlp.5", "node": "l:435", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 110, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 13, 2 ] ], "id": "L3.mlp.6", "node": "l:436", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 107, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 15, 2 ] ], "id": "L3.mlp.7", "node": "l:125", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 108, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L3.mlp.8", "node": "l:38", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 117, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.9", "node": "l:39", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 22, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.10", "node": "l:58", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 120, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.11", "node": "l:64", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 121, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.12", "node": "l:65", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 122, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.13", "node": "l:67", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 123, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.14", "node": "l:73", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 124, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.15", "node": "l:74", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 125, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.16", "node": "l:76", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 126, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.17", "node": "l:376", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 127, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.18", "node": "l:379", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 144, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.19", "node": "l:382", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 161, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.20", "node": "l:96", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 24, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L3.mlp.21", "node": "l:17", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 25, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 17, 2 ] ], "id": "L3.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 26, 1 ] ], "source_columns_b": null, "target_columns": [ [ 26, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 19, 2 ] ], "id": "L3.mlp.23", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 111, 1 ] ], "source_columns_b": null, "target_columns": [ [ 111, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 21, 2 ] ], "id": "L3.mlp.24", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 27, 1 ] ], "source_columns_b": null, "target_columns": [ [ 27, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 23, 2 ] ], "id": "L3.mlp.25", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 28, 1 ] ], "source_columns_b": null, "target_columns": [ [ 28, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:41", "l:38", "l:39", "l:58", "l:37", "l:40", "l:64", "l:65", "l:67", "l:63", "l:66", "l:73", "l:74", "l:76", "l:72", "l:75", "l:120", "l:96", "l:125", "l:156", "l:173", "l:94", "l:376", "l:379", "l:382", "l:17", "l:387", "l:388", "l:386", "l:415", "l:416", "l:414", "l:435", "l:436", "l:434" ] }, { "active_attention_heads": 8, "active_mlp_neurons": 310, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L4.attn.0", "key_source_columns": [ [ 33, 1 ], [ 32, 1 ] ], "node": "l:152", "query_source_columns": [ [ 30, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 1 ] ], "source_columns_b": null, "target_columns": [ [ 36, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L4.attn.1", "key_source_columns": [ [ 33, 1 ], [ 37, 1 ] ], "node": "l:116", "query_source_columns": [ [ 35, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 1 ] ], "source_columns_b": null, "target_columns": [ [ 37, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L4.attn.2", "key_source_columns": [ [ 33, 1 ], [ 48, 1 ] ], "node": "l:90", "query_source_columns": [ [ 46, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 46, 1 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L4.attn.3", "key_source_columns": [ [ 33, 1 ], [ 120, 1 ] ], "node": "l:36", "query_source_columns": [ [ 117, 1 ], [ 22, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ] ], "source_columns_b": null, "target_columns": [ [ 47, 1 ] ], "type": "compute_attn" }, { "heads": [ 4, 1 ], "id": "L4.attn.4", "key_source_columns": [ [ 33, 1 ], [ 123, 1 ] ], "node": "l:62", "query_source_columns": [ [ 121, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 1 ] ], "source_columns_b": null, "target_columns": [ [ 48, 1 ] ], "type": "compute_attn" }, { "heads": [ 5, 1 ], "id": "L4.attn.5", "key_source_columns": [ [ 33, 1 ], [ 126, 1 ] ], "node": "l:71", "query_source_columns": [ [ 124, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 20, 1 ] ], "source_columns_b": null, "target_columns": [ [ 18, 1 ] ], "type": "compute_attn" }, { "heads": [ 6, 1 ], "id": "L4.attn.6", "key_source_columns": [ [ 0, 1 ] ], "node": "l:126", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 108, 1 ] ], "source_columns_b": null, "target_columns": [ [ 19, 1 ] ], "type": "compute_linear" }, { "heads": [ 7, 1 ], "id": "L4.attn.7", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 18, 3 ], [ 22, 1 ], [ 30, 3 ], [ 35, 3 ], [ 46, 3 ], [ 117, 1 ], [ 120, 7 ] ], "type": "cancel" } ], "emitted_attention_heads": 8, "index": 4, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L4.mlp.0", "node": "l:115", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 37, 1 ] ], "source_columns_b": null, "target_columns": [ [ 20, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 12 ] ], "id": "L4.mlp.1", "node": "l:123", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 108, 1 ], [ 19, 1 ] ], "source_columns_b": null, "target_columns": [ [ 22, 1 ], [ 111, 1 ], [ 117, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 15, 2 ] ], "id": "L4.mlp.2", "node": "l:163", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 23, 1 ], [ 37, 1 ], [ 112, 1 ] ], "source_columns_b": null, "target_columns": [ [ 120, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 17, 4 ] ], "id": "L4.mlp.3", "node": "l:172", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 1 ], [ 42, 4 ] ], "source_columns_b": null, "target_columns": [ [ 121, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 21, 17 ] ], "id": "L4.mlp.4", "node": "l:227", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 46, 1 ] ], "source_columns_b": null, "target_columns": [ [ 26, 2 ], [ 125, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 38, 13 ] ], "id": "L4.mlp.5", "node": "l:201", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 46, 1 ] ], "source_columns_b": null, "target_columns": [ [ 28, 1 ], [ 30, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 51, 9 ] ], "id": "L4.mlp.6", "node": "l:89", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 46, 1 ] ], "source_columns_b": null, "target_columns": [ [ 35, 1 ], [ 178, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 60, 17 ] ], "id": "L4.mlp.7", "node": "l:374", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 47, 1 ], [ 127, 17 ] ], "source_columns_b": null, "target_columns": [ [ 181, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 77, 17 ] ], "id": "L4.mlp.8", "node": "l:377", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 48, 1 ], [ 144, 17 ] ], "source_columns_b": null, "target_columns": [ [ 198, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 94, 17 ] ], "id": "L4.mlp.9", "node": "l:380", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ], [ 161, 17 ] ], "source_columns_b": null, "target_columns": [ [ 215, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 111, 17 ] ], "id": "L4.mlp.10", "node": "l:395", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 47, 1 ], [ 127, 17 ] ], "source_columns_b": null, "target_columns": [ [ 232, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 128, 17 ] ], "id": "L4.mlp.11", "node": "l:397", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 48, 1 ], [ 144, 17 ] ], "source_columns_b": null, "target_columns": [ [ 249, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 145, 17 ] ], "id": "L4.mlp.12", "node": "l:399", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ], [ 161, 17 ] ], "source_columns_b": null, "target_columns": [ [ 266, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 162, 2 ] ], "id": "L4.mlp.13", "node": "l:16", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 283, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 164, 2 ] ], "id": "L4.mlp.14", "node": "l:22", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 284, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 166, 2 ] ], "id": "L4.mlp.15", "node": "l:366", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 285, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 168, 2 ] ], "id": "L4.mlp.16", "node": "l:367", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 286, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 170, 2 ] ], "id": "L4.mlp.17", "node": "l:405", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 287, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 172, 2 ] ], "id": "L4.mlp.18", "node": "l:406", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 288, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 174, 2 ] ], "id": "L4.mlp.19", "node": "l:425", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 289, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 176, 2 ] ], "id": "L4.mlp.20", "node": "l:426", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 290, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 178, 2 ] ], "id": "L4.mlp.21", "node": "l:445", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 291, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 180, 2 ] ], "id": "L4.mlp.22", "node": "l:446", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 292, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 182, 2 ] ], "id": "L4.mlp.23", "node": "l:385", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 38, 2 ] ], "source_columns_b": null, "target_columns": [ [ 293, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 184, 2 ] ], "id": "L4.mlp.24", "node": "l:413", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 40, 2 ] ], "source_columns_b": null, "target_columns": [ [ 294, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 186, 2 ] ], "id": "L4.mlp.25", "node": "l:433", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 110, 1 ], [ 107, 1 ] ], "source_columns_b": null, "target_columns": [ [ 295, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 188, 2 ] ], "id": "L4.mlp.26", "node": "l:158", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 33, 1 ] ], "source_columns_b": null, "target_columns": [ [ 296, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L4.mlp.27", "node": "l:59", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 297, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L4.mlp.28", "node": "l:68", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 314, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L4.mlp.29", "node": "l:126", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 19, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 190, 34 ] ], "id": "L4.mlp.30", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 127, 17 ] ], "source_columns_b": null, "target_columns": [ [ 127, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 224, 34 ] ], "id": "L4.mlp.31", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 144, 17 ] ], "source_columns_b": null, "target_columns": [ [ 144, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 258, 34 ] ], "id": "L4.mlp.32", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 161, 17 ] ], "source_columns_b": null, "target_columns": [ [ 161, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 292, 8 ] ], "id": "L4.mlp.33", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 42, 4 ] ], "source_columns_b": null, "target_columns": [ [ 42, 4 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 300, 2 ] ], "id": "L4.mlp.34", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 33, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 302, 2 ] ], "id": "L4.mlp.35", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 112, 1 ] ], "source_columns_b": null, "target_columns": [ [ 112, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 304, 2 ] ], "id": "L4.mlp.36", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 24, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 306, 2 ] ], "id": "L4.mlp.37", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 108, 1 ] ], "source_columns_b": null, "target_columns": [ [ 108, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 308, 2 ] ], "id": "L4.mlp.38", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 25, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:36", "l:62", "l:71", "l:116", "l:115", "l:113", "l:164", "l:163", "l:126", "l:124", "l:123", "l:152", "l:158", "l:172", "l:176", "l:90", "l:59", "l:68", "l:35", "l:61", "l:251", "l:253", "l:276", "l:375", "l:374", "l:378", "l:377", "l:381", "l:380", "l:373", "l:396", "l:395", "l:398", "l:397", "l:400", "l:399", "l:394", "l:16", "l:22", "l:15", "l:366", "l:367", "l:365", "l:385", "l:405", "l:406", "l:404", "l:413", "l:425", "l:426", "l:424", "l:433", "l:445", "l:446", "l:444", "l:227", "l:201", "l:89" ] }, { "active_attention_heads": 6, "active_mlp_neurons": 366, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L5.attn.0", "key_source_columns": [ [ 36, 1 ] ], "node": "l:150", "query_source_columns": [ [ 34, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 296, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L5.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:122", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 22, 1 ], [ 111, 1 ], [ 117, 1 ] ], "source_columns_b": null, "target_columns": [ [ 34, 1 ], [ 38, 2 ] ], "type": "compute_linear" }, { "heads": [ 2, 1 ], "id": "L5.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:372", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 181, 51 ] ], "source_columns_b": null, "target_columns": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "type": "compute_linear" }, { "heads": [ 3, 1 ], "id": "L5.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": "l:393", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 232, 51 ] ], "source_columns_b": null, "target_columns": [ [ 136, 17 ] ], "type": "compute_linear" }, { "heads": [ 4, 2 ], "id": "L5.attn.4", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 1 ], [ 36, 6 ], [ 46, 1 ], [ 107, 1 ], [ 110, 1 ], [ 181, 68 ], [ 296, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 6, "index": 5, "mlp_operations": [ { "hidden_slots": [ [ 1, 32 ] ], "id": "L5.mlp.0", "node": "l:149", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 1 ] ], "source_columns_b": null, "target_columns": [ [ 153, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 33, 2 ] ], "id": "L5.mlp.1", "node": "l:112", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 23, 1 ], [ 20, 1 ] ], "source_columns_b": null, "target_columns": [ [ 154, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 35, 34 ] ], "id": "L5.mlp.2", "node": "l:175", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 120, 1 ], [ 1, 17 ], [ 71, 17 ] ], "source_columns_b": null, "target_columns": [ [ 155, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 69, 17 ] ], "id": "L5.mlp.3", "node": "l:34", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 47, 1 ], [ 297, 17 ] ], "source_columns_b": null, "target_columns": [ [ 36, 2 ], [ 172, 6 ], [ 181, 9 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 86, 17 ] ], "id": "L5.mlp.4", "node": "l:60", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 48, 1 ], [ 314, 17 ] ], "source_columns_b": null, "target_columns": [ [ 190, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 103, 17 ] ], "id": "L5.mlp.5", "node": "l:250", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 47, 1 ], [ 297, 17 ] ], "source_columns_b": null, "target_columns": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 120, 17 ] ], "id": "L5.mlp.6", "node": "l:252", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 48, 1 ], [ 314, 17 ] ], "source_columns_b": null, "target_columns": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 137, 17 ] ], "id": "L5.mlp.7", "node": "l:275", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 48, 1 ], [ 314, 17 ] ], "source_columns_b": null, "target_columns": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 154, 2 ] ], "id": "L5.mlp.8", "node": "l:14", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 283, 2 ] ], "source_columns_b": null, "target_columns": [ [ 339, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 156, 2 ] ], "id": "L5.mlp.9", "node": "l:364", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 285, 2 ] ], "source_columns_b": null, "target_columns": [ [ 340, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 158, 2 ] ], "id": "L5.mlp.10", "node": "l:403", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 287, 2 ] ], "source_columns_b": null, "target_columns": [ [ 341, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 160, 2 ] ], "id": "L5.mlp.11", "node": "l:423", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 289, 2 ] ], "source_columns_b": null, "target_columns": [ [ 342, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 162, 2 ] ], "id": "L5.mlp.12", "node": "l:443", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 291, 2 ] ], "source_columns_b": null, "target_columns": [ [ 343, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L5.mlp.13", "node": "l:122", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 1 ], [ 38, 2 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 164, 34 ] ], "id": "L5.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 71, 17 ] ], "source_columns_b": null, "target_columns": [ [ 71, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 198, 34 ] ], "id": "L5.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 297, 17 ] ], "source_columns_b": null, "target_columns": [ [ 297, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 232, 34 ] ], "id": "L5.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 314, 17 ] ], "source_columns_b": null, "target_columns": [ [ 314, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 266, 34 ] ], "id": "L5.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 249, 17 ] ], "source_columns_b": null, "target_columns": [ [ 249, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 300, 34 ] ], "id": "L5.mlp.18", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 266, 17 ] ], "source_columns_b": null, "target_columns": [ [ 266, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 334, 6 ] ], "id": "L5.mlp.19", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 22, 1 ], [ 111, 1 ], [ 117, 1 ] ], "source_columns_b": null, "target_columns": [ [ 22, 1 ], [ 111, 1 ], [ 117, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 340, 2 ] ], "id": "L5.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 23, 1 ] ], "source_columns_b": null, "target_columns": [ [ 23, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 342, 2 ] ], "id": "L5.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 20, 1 ] ], "source_columns_b": null, "target_columns": [ [ 20, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 344, 2 ] ], "id": "L5.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 19, 1 ] ], "source_columns_b": null, "target_columns": [ [ 19, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 346, 2 ] ], "id": "L5.mlp.23", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 283, 1 ] ], "source_columns_b": null, "target_columns": [ [ 283, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 348, 2 ] ], "id": "L5.mlp.24", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 284, 1 ] ], "source_columns_b": null, "target_columns": [ [ 284, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 350, 2 ] ], "id": "L5.mlp.25", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 285, 1 ] ], "source_columns_b": null, "target_columns": [ [ 285, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 352, 2 ] ], "id": "L5.mlp.26", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 286, 1 ] ], "source_columns_b": null, "target_columns": [ [ 286, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 354, 2 ] ], "id": "L5.mlp.27", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 287, 1 ] ], "source_columns_b": null, "target_columns": [ [ 287, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 356, 2 ] ], "id": "L5.mlp.28", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 288, 1 ] ], "source_columns_b": null, "target_columns": [ [ 288, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 358, 2 ] ], "id": "L5.mlp.29", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 289, 1 ] ], "source_columns_b": null, "target_columns": [ [ 289, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 360, 2 ] ], "id": "L5.mlp.30", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 290, 1 ] ], "source_columns_b": null, "target_columns": [ [ 290, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 362, 2 ] ], "id": "L5.mlp.31", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 291, 1 ] ], "source_columns_b": null, "target_columns": [ [ 291, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 364, 2 ] ], "id": "L5.mlp.32", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 292, 1 ] ], "source_columns_b": null, "target_columns": [ [ 292, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:112", "l:122", "l:111", "l:132", "l:150", "l:149", "l:175", "l:34", "l:60", "l:250", "l:252", "l:275", "l:372", "l:393", "l:450", "l:14", "l:364", "l:403", "l:423", "l:443" ] }, { "active_attention_heads": 2, "active_mlp_neurons": 84, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L6.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:148", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 153, 1 ] ], "source_columns_b": null, "target_columns": [ [ 71, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L6.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 33, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 2, "index": 6, "mlp_operations": [ { "hidden_slots": [ [ 1, 3 ] ], "id": "L6.mlp.0", "node": "l:110", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 154, 1 ], [ 34, 1 ], [ 38, 2 ] ], "source_columns_b": null, "target_columns": [ [ 72, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 4, 34 ] ], "id": "L6.mlp.1", "node": "l:131", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 154, 1 ], [ 1, 17 ], [ 53, 17 ] ], "source_columns_b": null, "target_columns": [ [ 19, 2 ], [ 75, 13 ], [ 111, 1 ], [ 329, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 38, 2 ] ], "id": "L6.mlp.2", "node": "l:169", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 71, 1 ] ], "source_columns_b": null, "target_columns": [ [ 22, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L6.mlp.3", "node": "l:148", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 71, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 40, 34 ] ], "id": "L6.mlp.4", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 53, 17 ] ], "source_columns_b": null, "target_columns": [ [ 53, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 74, 6 ] ], "id": "L6.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 34, 1 ], [ 38, 2 ] ], "source_columns_b": null, "target_columns": [ [ 34, 1 ], [ 38, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 80, 2 ] ], "id": "L6.mlp.6", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 154, 1 ] ], "source_columns_b": null, "target_columns": [ [ 154, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 82, 2 ] ], "id": "L6.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 153, 1 ] ], "source_columns_b": null, "target_columns": [ [ 153, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:110", "l:109", "l:131", "l:148", "l:169" ] }, { "active_attention_heads": 4, "active_mlp_neurons": 56, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L7.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:108", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 1, "role": "transport", "source_columns": [ [ 72, 3 ], [ 52, 1 ] ], "source_columns_b": null, "target_columns": [ [ 113, 4 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L7.attn.1", "key_source_columns": [ [ 21, 1 ], [ 51, 1 ] ], "node": "l:142", "query_source_columns": [ [ 49, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 71, 1 ] ], "source_columns_b": null, "target_columns": [ [ 51, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L7.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:170", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 22, 1 ] ], "source_columns_b": null, "target_columns": [ [ 53, 1 ] ], "type": "compute_linear" }, { "heads": [ 3, 1 ], "id": "L7.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1, 17 ], [ 49, 3 ] ], "type": "cancel" } ], "emitted_attention_heads": 4, "index": 7, "mlp_operations": [ { "hidden_slots": [ [ 1, 12 ] ], "id": "L7.mlp.0", "node": "l:167", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 22, 1 ], [ 53, 1 ] ], "source_columns_b": null, "target_columns": [ [ 54, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 13, 17 ] ], "id": "L7.mlp.1", "node": "l:215", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 51, 1 ] ], "source_columns_b": null, "target_columns": [ [ 57, 2 ], [ 61, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 30, 13 ] ], "id": "L7.mlp.2", "node": "l:191", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 51, 1 ] ], "source_columns_b": null, "target_columns": [ [ 63, 2 ], [ 68, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 43, 9 ] ], "id": "L7.mlp.3", "node": "l:141", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 51, 1 ] ], "source_columns_b": null, "target_columns": [ [ 23, 1 ], [ 66, 1 ], [ 117, 1 ], [ 330, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L7.mlp.4", "node": "l:170", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 53, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 52, 2 ] ], "id": "L7.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 71, 1 ] ], "source_columns_b": null, "target_columns": [ [ 71, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 54, 2 ] ], "id": "L7.mlp.6", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 22, 1 ] ], "source_columns_b": null, "target_columns": [ [ 22, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:108", "l:170", "l:168", "l:167", "l:142", "l:215", "l:191", "l:141" ] }, { "active_attention_heads": 5, "active_mlp_neurons": 22, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L8.attn.0", "key_source_columns": [ [ 113, 4 ] ], "node": "l:88", "query_source_columns": [ [ 35, 1 ], [ 178, 3 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 2 ], [ 75, 13 ], [ 111, 1 ], [ 329, 1 ] ], "source_columns_b": null, "target_columns": [ [ 19, 1 ], [ 51, 1 ], [ 71, 1 ], [ 75, 13 ], [ 329, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L8.attn.1", "key_source_columns": [ [ 113, 4 ] ], "node": "l:200", "query_source_columns": [ [ 28, 1 ], [ 30, 3 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 2 ], [ 75, 13 ], [ 111, 1 ], [ 329, 1 ] ], "source_columns_b": null, "target_columns": [ [ 20, 1 ], [ 22, 1 ], [ 26, 3 ], [ 30, 3 ], [ 111, 1 ], [ 113, 4 ], [ 125, 2 ], [ 153, 2 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L8.attn.2", "key_source_columns": [ [ 113, 4 ] ], "node": "l:226", "query_source_columns": [ [ 26, 2 ], [ 125, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 2 ], [ 75, 13 ], [ 111, 1 ], [ 329, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 3 ], [ 38, 2 ], [ 49, 2 ], [ 178, 3 ], [ 249, 6 ], [ 327, 1 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L8.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": "l:166", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 54, 3 ] ], "source_columns_b": null, "target_columns": [ [ 255, 3 ] ], "type": "compute_linear" }, { "heads": [ 4, 1 ], "id": "L8.attn.4", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 19, 2 ], [ 26, 3 ], [ 30, 3 ], [ 35, 1 ], [ 51, 1 ], [ 75, 13 ], [ 111, 1 ], [ 113, 4 ], [ 125, 2 ], [ 178, 3 ], [ 329, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 5, "index": 8, "mlp_operations": [ { "hidden_slots": [ [ 1, 3 ] ], "id": "L8.mlp.0", "node": "l:161", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 120, 1 ], [ 255, 3 ] ], "source_columns_b": null, "target_columns": [ [ 258, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L8.mlp.1", "node": "l:166", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 255, 3 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 4, 6 ] ], "id": "L8.mlp.2", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 72, 3 ] ], "source_columns_b": null, "target_columns": [ [ 72, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 10, 6 ] ], "id": "L8.mlp.3", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 54, 3 ] ], "source_columns_b": null, "target_columns": [ [ 54, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 16, 2 ] ], "id": "L8.mlp.4", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 120, 1 ] ], "source_columns_b": null, "target_columns": [ [ 120, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 18, 2 ] ], "id": "L8.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 52, 1 ] ], "source_columns_b": null, "target_columns": [ [ 52, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 20, 2 ] ], "id": "L8.mlp.6", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 53, 1 ] ], "source_columns_b": null, "target_columns": [ [ 53, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:166", "l:162", "l:161", "l:160", "l:88", "l:200", "l:226" ] }, { "active_attention_heads": 1, "active_mlp_neurons": 7, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L9.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:159", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 1, "role": "transport", "source_columns": [ [ 258, 3 ], [ 70, 1 ] ], "source_columns_b": null, "target_columns": [ [ 121, 4 ] ], "type": "add_into" } ], "emitted_attention_heads": 1, "index": 9, "mlp_operations": [ { "hidden_slots": [ [ 1, 6 ] ], "id": "L9.mlp.0", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 255, 3 ] ], "source_columns_b": null, "target_columns": [ [ 255, 3 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:159" ] }, { "active_attention_heads": 4, "active_mlp_neurons": 9, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L10.attn.0", "key_source_columns": [ [ 121, 4 ] ], "node": "l:140", "query_source_columns": [ [ 23, 1 ], [ 66, 1 ], [ 117, 1 ], [ 330, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 155, 17 ] ], "source_columns_b": null, "target_columns": [ [ 23, 1 ], [ 54, 5 ], [ 61, 4 ], [ 68, 2 ], [ 72, 3 ], [ 117, 1 ], [ 330, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L10.attn.1", "key_source_columns": [ [ 121, 4 ] ], "node": "l:190", "query_source_columns": [ [ 63, 2 ], [ 68, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 155, 17 ] ], "source_columns_b": null, "target_columns": [ [ 66, 1 ], [ 120, 5 ], [ 155, 11 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L10.attn.2", "key_source_columns": [ [ 121, 4 ] ], "node": "l:214", "query_source_columns": [ [ 57, 2 ], [ 61, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 155, 17 ] ], "source_columns_b": null, "target_columns": [ [ 52, 2 ], [ 166, 6 ], [ 255, 3 ], [ 261, 6 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L10.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 23, 1 ], [ 57, 2 ], [ 61, 4 ], [ 66, 1 ], [ 68, 2 ], [ 117, 1 ], [ 121, 4 ], [ 155, 17 ], [ 330, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 4, "index": 10, "mlp_operations": [ { "hidden_slots": [], "id": "L10.mlp.0", "node": "l:274", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 267, 2 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 1, 6 ] ], "id": "L10.mlp.1", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 258, 3 ] ], "source_columns_b": null, "target_columns": [ [ 258, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 7, 2 ] ], "id": "L10.mlp.2", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 70, 1 ] ], "source_columns_b": null, "target_columns": [ [ 70, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:140", "l:190", "l:214", "l:274" ] }, { "active_attention_heads": 8, "active_mlp_neurons": 1605, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L11.attn.0", "key_source_columns": [ [ 21, 1 ], [ 90, 1 ] ], "node": "l:82", "query_source_columns": [ [ 88, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 1 ], [ 51, 1 ], [ 71, 1 ], [ 75, 13 ], [ 329, 1 ] ], "source_columns_b": null, "target_columns": [ [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L11.attn.1", "key_source_columns": [ [ 21, 1 ], [ 93, 1 ] ], "node": "l:194", "query_source_columns": [ [ 91, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 20, 1 ], [ 22, 1 ], [ 26, 3 ], [ 30, 3 ], [ 111, 1 ], [ 113, 4 ], [ 125, 2 ], [ 153, 2 ] ], "source_columns_b": null, "target_columns": [ [ 76, 13 ], [ 90, 3 ], [ 329, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L11.attn.2", "key_source_columns": [ [ 21, 1 ], [ 96, 1 ] ], "node": "l:220", "query_source_columns": [ [ 94, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 3 ], [ 38, 2 ], [ 49, 2 ], [ 178, 3 ], [ 249, 6 ], [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L11.attn.3", "key_source_columns": [ [ 21, 1 ], [ 99, 1 ] ], "node": "l:134", "query_source_columns": [ [ 97, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 23, 1 ], [ 54, 5 ], [ 61, 4 ], [ 68, 2 ], [ 72, 3 ], [ 117, 1 ], [ 330, 1 ] ], "source_columns_b": null, "target_columns": [ [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "type": "compute_attn" }, { "heads": [ 4, 1 ], "id": "L11.attn.4", "key_source_columns": [ [ 21, 1 ], [ 102, 1 ] ], "node": "l:184", "query_source_columns": [ [ 100, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 66, 1 ], [ 120, 5 ], [ 155, 11 ] ], "source_columns_b": null, "target_columns": [ [ 30, 3 ], [ 153, 14 ] ], "type": "compute_attn" }, { "heads": [ 5, 1 ], "id": "L11.attn.5", "key_source_columns": [ [ 21, 1 ], [ 105, 1 ] ], "node": "l:208", "query_source_columns": [ [ 103, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 52, 2 ], [ 166, 6 ], [ 255, 3 ], [ 261, 6 ] ], "source_columns_b": null, "target_columns": [ [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ] ], "type": "compute_attn" }, { "heads": [ 6, 2 ], "id": "L11.attn.6", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 19, 5 ], [ 26, 3 ], [ 30, 6 ], [ 38, 2 ], [ 49, 10 ], [ 61, 4 ], [ 66, 1 ], [ 68, 2 ], [ 71, 35 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 153, 19 ], [ 178, 3 ], [ 249, 9 ], [ 261, 6 ], [ 327, 1 ], [ 329, 2 ] ], "type": "cancel" } ], "emitted_attention_heads": 8, "index": 11, "mlp_operations": [ { "hidden_slots": [ [ 1, 100 ] ], "id": "L11.mlp.0", "node": "l:300", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ], [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "source_columns_b": null, "target_columns": [ [ 250, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 101, 100 ] ], "id": "L11.mlp.1", "node": "l:304", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 76, 13 ], [ 90, 3 ], [ 329, 1 ], [ 30, 3 ], [ 153, 14 ] ], "source_columns_b": null, "target_columns": [ [ 251, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 201, 100 ] ], "id": "L11.mlp.2", "node": "l:308", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ], [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 301, 100 ] ], "id": "L11.mlp.3", "node": "l:232", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 76, 13 ], [ 90, 3 ], [ 329, 1 ], [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 253, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 401, 100 ] ], "id": "L11.mlp.4", "node": "l:234", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ], [ 30, 3 ], [ 153, 14 ] ], "source_columns_b": null, "target_columns": [ [ 256, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 501, 100 ] ], "id": "L11.mlp.5", "node": "l:240", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ], [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 51, 1 ], [ 258, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 601, 100 ] ], "id": "L11.mlp.6", "node": "l:206", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ], [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 259, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 701, 100 ] ], "id": "L11.mlp.7", "node": "l:216", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 76, 13 ], [ 90, 3 ], [ 329, 1 ], [ 30, 3 ], [ 153, 14 ] ], "source_columns_b": null, "target_columns": [ [ 261, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 801, 100 ] ], "id": "L11.mlp.8", "node": "l:218", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ], [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "source_columns_b": null, "target_columns": [ [ 263, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 901, 100 ] ], "id": "L11.mlp.9", "node": "l:182", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ], [ 30, 3 ], [ 153, 14 ] ], "source_columns_b": null, "target_columns": [ [ 265, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1001, 100 ] ], "id": "L11.mlp.10", "node": "l:192", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 76, 13 ], [ 90, 3 ], [ 329, 1 ], [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "source_columns_b": null, "target_columns": [ [ 53, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1101, 200 ] ], "id": "L11.mlp.11", "node": "l:272", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ], [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ], [ 267, 2 ] ], "source_columns_b": null, "target_columns": [ [ 269, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1301, 100 ] ], "id": "L11.mlp.12", "node": "l:80", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ], [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "source_columns_b": null, "target_columns": [ [ 271, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1401, 200 ] ], "id": "L11.mlp.13", "node": "l:353", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ], [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ], [ 267, 2 ] ], "source_columns_b": null, "target_columns": [ [ 273, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1601, 4 ] ], "id": "L11.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 267, 2 ] ], "source_columns_b": null, "target_columns": [ [ 267, 2 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:82", "l:194", "l:220", "l:134", "l:184", "l:208", "l:273", "l:353", "l:272", "l:271", "l:301", "l:300", "l:305", "l:304", "l:309", "l:308", "l:81", "l:80", "l:183", "l:182", "l:207", "l:206", "l:193", "l:192", "l:217", "l:216", "l:233", "l:232", "l:219", "l:218", "l:235", "l:234", "l:241", "l:240", "l:239", "l:231", "l:205", "l:181", "l:352" ] }, { "active_attention_heads": 2, "active_mlp_neurons": 501, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L12.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:238", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 253, 2 ], [ 256, 2 ], [ 51, 1 ], [ 258, 1 ] ], "source_columns_b": null, "target_columns": [ [ 267, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L12.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:360", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 51, 1 ], [ 258, 1 ] ], "source_columns_b": null, "target_columns": [ [ 268, 1 ] ], "type": "compute_linear" } ], "emitted_attention_heads": 2, "index": 12, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L12.mlp.0", "node": "l:299", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 250, 1 ] ], "source_columns_b": null, "target_columns": [ [ 290, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L12.mlp.1", "node": "l:302", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 250, 1 ] ], "source_columns_b": null, "target_columns": [ [ 291, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 2 ] ], "id": "L12.mlp.2", "node": "l:303", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 251, 1 ] ], "source_columns_b": null, "target_columns": [ [ 292, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 7, 2 ] ], "id": "L12.mlp.3", "node": "l:306", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 251, 1 ] ], "source_columns_b": null, "target_columns": [ [ 60, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 9, 2 ] ], "id": "L12.mlp.4", "node": "l:307", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 50, 1 ] ], "source_columns_b": null, "target_columns": [ [ 59, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 11, 2 ] ], "id": "L12.mlp.5", "node": "l:310", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 50, 1 ] ], "source_columns_b": null, "target_columns": [ [ 297, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 13, 200 ] ], "id": "L12.mlp.6", "node": "l:270", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 76, 13 ], [ 90, 3 ], [ 329, 1 ], [ 30, 3 ], [ 153, 14 ], [ 269, 2 ] ], "source_columns_b": null, "target_columns": [ [ 298, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 213, 200 ] ], "id": "L12.mlp.7", "node": "l:337", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 76, 13 ], [ 90, 3 ], [ 329, 1 ], [ 30, 3 ], [ 153, 14 ], [ 269, 2 ] ], "source_columns_b": null, "target_columns": [ [ 300, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 413, 38 ] ], "id": "L12.mlp.8", "node": "l:359", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 268, 1 ] ], "source_columns_b": null, "target_columns": [ [ 65, 1 ], [ 67, 1 ], [ 317, 2 ], [ 320, 7 ], [ 328, 1 ], [ 344, 5 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 451, 2 ] ], "id": "L12.mlp.9", "node": "l:230", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 259, 4 ], [ 253, 2 ], [ 263, 2 ], [ 256, 2 ] ], "source_columns_b": null, "target_columns": [ [ 349, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 453, 2 ] ], "id": "L12.mlp.10", "node": "l:204", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 265, 2 ], [ 259, 2 ], [ 53, 2 ], [ 261, 4 ] ], "source_columns_b": null, "target_columns": [ [ 350, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 455, 2 ] ], "id": "L12.mlp.11", "node": "l:180", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 271, 2 ], [ 265, 2 ], [ 53, 2 ] ], "source_columns_b": null, "target_columns": [ [ 351, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 457, 2 ] ], "id": "L12.mlp.12", "node": "l:79", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 271, 2 ] ], "source_columns_b": null, "target_columns": [ [ 352, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 459, 4 ] ], "id": "L12.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 269, 2 ] ], "source_columns_b": null, "target_columns": [ [ 269, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 463, 4 ] ], "id": "L12.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 271, 2 ] ], "source_columns_b": null, "target_columns": [ [ 271, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 467, 4 ] ], "id": "L12.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 265, 2 ] ], "source_columns_b": null, "target_columns": [ [ 265, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 471, 4 ] ], "id": "L12.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 259, 2 ] ], "source_columns_b": null, "target_columns": [ [ 259, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 475, 4 ] ], "id": "L12.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 53, 2 ] ], "source_columns_b": null, "target_columns": [ [ 53, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 479, 4 ] ], "id": "L12.mlp.18", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 253, 2 ] ], "source_columns_b": null, "target_columns": [ [ 253, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 483, 4 ] ], "id": "L12.mlp.19", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 263, 2 ] ], "source_columns_b": null, "target_columns": [ [ 263, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 487, 4 ] ], "id": "L12.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 256, 2 ] ], "source_columns_b": null, "target_columns": [ [ 256, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 491, 4 ] ], "id": "L12.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 51, 1 ], [ 258, 1 ] ], "source_columns_b": null, "target_columns": [ [ 51, 1 ], [ 258, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 495, 2 ] ], "id": "L12.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 250, 1 ] ], "source_columns_b": null, "target_columns": [ [ 250, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 497, 2 ] ], "id": "L12.mlp.23", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 251, 1 ] ], "source_columns_b": null, "target_columns": [ [ 251, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 499, 2 ] ], "id": "L12.mlp.24", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 50, 1 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:337", "l:270", "l:269", "l:299", "l:302", "l:303", "l:306", "l:307", "l:310", "l:298", "l:360", "l:238", "l:230", "l:204", "l:180", "l:79", "l:336", "l:358", "l:359" ] }, { "active_attention_heads": 3, "active_mlp_neurons": 643, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L13.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:237", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 106, 1 ] ], "source_columns_b": null, "target_columns": [ [ 267, 1 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L13.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:297", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 290, 3 ], [ 60, 1 ], [ 59, 1 ], [ 297, 1 ] ], "source_columns_b": null, "target_columns": [ [ 250, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 1 ], "id": "L13.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 261, 2 ], [ 268, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 3, "index": 13, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L13.mlp.0", "node": "l:296", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 250, 1 ] ], "source_columns_b": null, "target_columns": [ [ 251, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 12 ] ], "id": "L13.mlp.1", "node": "l:236", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 267, 1 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 15, 200 ] ], "id": "L13.mlp.2", "node": "l:268", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ], [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ], [ 298, 2 ] ], "source_columns_b": null, "target_columns": [ [ 253, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 215, 200 ] ], "id": "L13.mlp.3", "node": "l:289", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ], [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ], [ 298, 2 ] ], "source_columns_b": null, "target_columns": [ [ 51, 1 ], [ 53, 2 ], [ 256, 11 ], [ 269, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 415, 110 ] ], "id": "L13.mlp.4", "node": "l:343", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 267, 1 ] ], "source_columns_b": null, "target_columns": [ [ 272, 1 ], [ 353, 16 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 525, 17 ] ], "id": "L13.mlp.5", "node": "l:335", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 47, 1 ], [ 300, 17 ] ], "source_columns_b": null, "target_columns": [ [ 369, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 542, 17 ] ], "id": "L13.mlp.6", "node": "l:357", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ], [ 65, 1 ], [ 67, 1 ], [ 317, 2 ], [ 320, 7 ], [ 328, 1 ], [ 344, 5 ] ], "source_columns_b": null, "target_columns": [ [ 386, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L13.mlp.7", "node": "l:267", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 403, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 559, 34 ] ], "id": "L13.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 300, 17 ] ], "source_columns_b": null, "target_columns": [ [ 300, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 593, 34 ] ], "id": "L13.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 65, 1 ], [ 67, 1 ], [ 317, 2 ], [ 320, 7 ], [ 328, 1 ], [ 344, 5 ] ], "source_columns_b": null, "target_columns": [ [ 65, 1 ], [ 67, 1 ], [ 317, 2 ], [ 320, 7 ], [ 328, 1 ], [ 344, 5 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 627, 4 ] ], "id": "L13.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 298, 2 ] ], "source_columns_b": null, "target_columns": [ [ 298, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 631, 2 ] ], "id": "L13.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 290, 1 ] ], "source_columns_b": null, "target_columns": [ [ 290, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 633, 2 ] ], "id": "L13.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 291, 1 ] ], "source_columns_b": null, "target_columns": [ [ 291, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 635, 2 ] ], "id": "L13.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 292, 1 ] ], "source_columns_b": null, "target_columns": [ [ 292, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 637, 2 ] ], "id": "L13.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 60, 1 ] ], "source_columns_b": null, "target_columns": [ [ 60, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 639, 2 ] ], "id": "L13.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 59, 1 ] ], "source_columns_b": null, "target_columns": [ [ 59, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 641, 2 ] ], "id": "L13.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 297, 1 ] ], "source_columns_b": null, "target_columns": [ [ 297, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:267", "l:289", "l:268", "l:266", "l:297", "l:296", "l:295", "l:316", "l:322", "l:312", "l:318", "l:324", "l:237", "l:236", "l:288", "l:335", "l:342", "l:357", "l:343" ] }, { "active_attention_heads": 2, "active_mlp_neurons": 922, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L14.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:229", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 50, 1 ] ], "source_columns_b": null, "target_columns": [ [ 349, 1 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L14.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 106, 1 ], [ 267, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 2, "index": 14, "mlp_operations": [ { "hidden_slots": [ [ 1, 34 ] ], "id": "L14.mlp.0", "node": "l:321", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 251, 1 ], [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ], [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 59, 2 ], [ 67, 1 ], [ 106, 1 ], [ 267, 1 ], [ 290, 3 ], [ 297, 9 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 35, 34 ] ], "id": "L14.mlp.1", "node": "l:323", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 251, 1 ], [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ], [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ] ], "source_columns_b": null, "target_columns": [ [ 306, 13 ], [ 320, 3 ], [ 328, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 69, 12 ] ], "id": "L14.mlp.2", "node": "l:228", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 349, 1 ] ], "source_columns_b": null, "target_columns": [ [ 323, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 81, 34 ] ], "id": "L14.mlp.3", "node": "l:315", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 251, 1 ], [ 76, 13 ], [ 90, 3 ], [ 329, 1 ], [ 30, 3 ], [ 153, 14 ] ], "source_columns_b": null, "target_columns": [ [ 65, 1 ], [ 324, 3 ], [ 344, 5 ], [ 420, 8 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 115, 34 ] ], "id": "L14.mlp.4", "node": "l:317", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 251, 1 ], [ 30, 3 ], [ 153, 14 ], [ 76, 13 ], [ 90, 3 ], [ 329, 1 ] ], "source_columns_b": null, "target_columns": [ [ 428, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 149, 34 ] ], "id": "L14.mlp.5", "node": "l:294", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 251, 1 ], [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ], [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "source_columns_b": null, "target_columns": [ [ 445, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 183, 34 ] ], "id": "L14.mlp.6", "node": "l:311", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 251, 1 ], [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ], [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ] ], "source_columns_b": null, "target_columns": [ [ 462, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 217, 200 ] ], "id": "L14.mlp.7", "node": "l:265", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 403, 17 ], [ 403, 17 ], [ 253, 2 ] ], "source_columns_b": null, "target_columns": [ [ 479, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 417, 206 ] ], "id": "L14.mlp.8", "node": "l:328", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 349, 1 ] ], "source_columns_b": null, "target_columns": [ [ 496, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 623, 17 ] ], "id": "L14.mlp.9", "node": "l:287", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 47, 1 ], [ 51, 1 ], [ 53, 2 ], [ 256, 11 ], [ 269, 3 ] ], "source_columns_b": null, "target_columns": [ [ 513, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 640, 2 ] ], "id": "L14.mlp.10", "node": "l:347", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 251, 1 ] ], "source_columns_b": null, "target_columns": [ [ 530, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L14.mlp.11", "node": "l:325", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 531, 2 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 642, 34 ] ], "id": "L14.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ] ], "source_columns_b": null, "target_columns": [ [ 52, 1 ], [ 55, 4 ], [ 61, 4 ], [ 68, 8 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 676, 34 ] ], "id": "L14.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 76, 13 ], [ 90, 3 ], [ 329, 1 ] ], "source_columns_b": null, "target_columns": [ [ 76, 13 ], [ 90, 3 ], [ 329, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 710, 34 ] ], "id": "L14.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ] ], "source_columns_b": null, "target_columns": [ [ 19, 5 ], [ 93, 3 ], [ 97, 4 ], [ 102, 4 ], [ 111, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 744, 34 ] ], "id": "L14.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "source_columns_b": null, "target_columns": [ [ 26, 3 ], [ 66, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 778, 34 ] ], "id": "L14.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 30, 3 ], [ 153, 14 ] ], "source_columns_b": null, "target_columns": [ [ 30, 3 ], [ 153, 14 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 812, 34 ] ], "id": "L14.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 3 ], [ 38, 2 ], [ 49, 1 ], [ 168, 4 ], [ 178, 3 ], [ 249, 1 ], [ 252, 1 ], [ 255, 1 ], [ 327, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 846, 34 ] ], "id": "L14.mlp.18", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 403, 17 ] ], "source_columns_b": null, "target_columns": [ [ 403, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 880, 34 ] ], "id": "L14.mlp.19", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 51, 1 ], [ 53, 2 ], [ 256, 11 ], [ 269, 3 ] ], "source_columns_b": null, "target_columns": [ [ 51, 1 ], [ 53, 2 ], [ 256, 11 ], [ 269, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 914, 4 ] ], "id": "L14.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 253, 2 ] ], "source_columns_b": null, "target_columns": [ [ 253, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 918, 2 ] ], "id": "L14.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 250, 1 ] ], "source_columns_b": null, "target_columns": [ [ 250, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 920, 2 ] ], "id": "L14.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 251, 1 ] ], "source_columns_b": null, "target_columns": [ [ 251, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:265", "l:294", "l:315", "l:321", "l:311", "l:317", "l:323", "l:325", "l:320", "l:229", "l:228", "l:264", "l:287", "l:327", "l:347", "l:346", "l:328" ] }, { "active_attention_heads": 2, "active_mlp_neurons": 797, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L15.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:203", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 323, 1 ] ], "source_columns_b": null, "target_columns": [ [ 350, 1 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L15.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 50, 1 ], [ 349, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 2, "index": 15, "mlp_operations": [ { "hidden_slots": [ [ 1, 200 ] ], "id": "L15.mlp.0", "node": "l:319", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 59, 2 ], [ 67, 1 ], [ 106, 1 ], [ 267, 1 ], [ 290, 3 ], [ 297, 22 ], [ 320, 3 ], [ 328, 1 ], [ 531, 2 ] ], "source_columns_b": null, "target_columns": [ [ 21, 1 ], [ 116, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 201, 12 ] ], "id": "L15.mlp.1", "node": "l:202", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 350, 1 ] ], "source_columns_b": null, "target_columns": [ [ 22, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 213, 206 ] ], "id": "L15.mlp.2", "node": "l:279", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 350, 1 ] ], "source_columns_b": null, "target_columns": [ [ 23, 1 ], [ 26, 2 ], [ 66, 1 ], [ 111, 1 ], [ 113, 3 ], [ 117, 1 ], [ 120, 7 ], [ 330, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 419, 17 ] ], "id": "L15.mlp.3", "node": "l:263", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 47, 1 ], [ 479, 17 ] ], "source_columns_b": null, "target_columns": [ [ 28, 1 ], [ 30, 3 ], [ 153, 13 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 436, 17 ] ], "id": "L15.mlp.4", "node": "l:326", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ], [ 496, 17 ] ], "source_columns_b": null, "target_columns": [ [ 33, 3 ], [ 38, 2 ], [ 50, 1 ], [ 166, 1 ], [ 168, 4 ], [ 178, 3 ], [ 251, 1 ], [ 327, 1 ], [ 404, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 453, 200 ] ], "id": "L15.mlp.5", "node": "l:356", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 59, 2 ], [ 67, 1 ], [ 106, 1 ], [ 267, 1 ], [ 290, 3 ], [ 297, 22 ], [ 320, 3 ], [ 328, 1 ], [ 531, 2 ] ], "source_columns_b": null, "target_columns": [ [ 49, 1 ], [ 51, 2 ], [ 249, 2 ], [ 252, 12 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 653, 2 ] ], "id": "L15.mlp.6", "node": "l:345", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 48, 1 ], [ 530, 1 ] ], "source_columns_b": null, "target_columns": [ [ 264, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 655, 34 ] ], "id": "L15.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 479, 17 ] ], "source_columns_b": null, "target_columns": [ [ 479, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 689, 34 ] ], "id": "L15.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 59, 2 ], [ 67, 1 ], [ 106, 1 ], [ 267, 1 ], [ 290, 3 ], [ 297, 9 ] ], "source_columns_b": null, "target_columns": [ [ 59, 2 ], [ 67, 1 ], [ 106, 1 ], [ 267, 1 ], [ 290, 3 ], [ 297, 9 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 723, 34 ] ], "id": "L15.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 306, 13 ], [ 320, 3 ], [ 328, 1 ] ], "source_columns_b": null, "target_columns": [ [ 306, 13 ], [ 320, 3 ], [ 328, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 757, 34 ] ], "id": "L15.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 496, 17 ] ], "source_columns_b": null, "target_columns": [ [ 496, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 791, 4 ] ], "id": "L15.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 531, 2 ] ], "source_columns_b": null, "target_columns": [ [ 531, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 795, 2 ] ], "id": "L15.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 530, 1 ] ], "source_columns_b": null, "target_columns": [ [ 530, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:356", "l:319", "l:314", "l:203", "l:202", "l:263", "l:278", "l:326", "l:355", "l:345", "l:279" ] }, { "active_attention_heads": 2, "active_mlp_neurons": 789, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L16.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:179", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 22, 1 ] ], "source_columns_b": null, "target_columns": [ [ 351, 1 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L16.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 323, 1 ], [ 350, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 2, "index": 16, "mlp_operations": [ { "hidden_slots": [ [ 1, 200 ] ], "id": "L16.mlp.0", "node": "l:313", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 65, 1 ], [ 324, 3 ], [ 344, 5 ], [ 420, 25 ], [ 21, 1 ], [ 116, 1 ] ], "source_columns_b": null, "target_columns": [ [ 106, 1 ], [ 267, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 201, 12 ] ], "id": "L16.mlp.1", "node": "l:178", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 351, 1 ] ], "source_columns_b": null, "target_columns": [ [ 67, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 213, 134 ] ], "id": "L16.mlp.2", "node": "l:256", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 351, 1 ] ], "source_columns_b": null, "target_columns": [ [ 53, 8 ], [ 265, 2 ], [ 269, 3 ], [ 290, 3 ], [ 297, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 347, 200 ] ], "id": "L16.mlp.3", "node": "l:340", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 65, 1 ], [ 324, 3 ], [ 344, 5 ], [ 420, 25 ], [ 21, 1 ], [ 116, 1 ] ], "source_columns_b": null, "target_columns": [ [ 61, 2 ], [ 298, 9 ], [ 308, 4 ], [ 313, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 547, 17 ] ], "id": "L16.mlp.4", "node": "l:277", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ], [ 23, 1 ], [ 26, 2 ], [ 66, 1 ], [ 111, 1 ], [ 113, 3 ], [ 117, 1 ], [ 120, 7 ], [ 330, 1 ] ], "source_columns_b": null, "target_columns": [ [ 63, 2 ], [ 70, 6 ], [ 315, 2 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 349, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 564, 17 ] ], "id": "L16.mlp.5", "node": "l:341", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ], [ 272, 1 ], [ 353, 16 ] ], "source_columns_b": null, "target_columns": [ [ 68, 1 ], [ 76, 7 ], [ 403, 1 ], [ 405, 8 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 581, 17 ] ], "id": "L16.mlp.6", "node": "l:351", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 47, 1 ], [ 273, 17 ] ], "source_columns_b": null, "target_columns": [ [ 83, 6 ], [ 90, 5 ], [ 329, 1 ], [ 414, 1 ], [ 416, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 598, 17 ] ], "id": "L16.mlp.7", "node": "l:354", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 48, 2 ], [ 51, 2 ], [ 249, 2 ], [ 252, 12 ] ], "source_columns_b": null, "target_columns": [ [ 19, 1 ], [ 95, 1 ], [ 97, 1 ], [ 479, 10 ], [ 490, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 615, 34 ] ], "id": "L16.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 273, 17 ] ], "source_columns_b": null, "target_columns": [ [ 273, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 649, 34 ] ], "id": "L16.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 65, 1 ], [ 324, 3 ], [ 344, 5 ], [ 420, 8 ] ], "source_columns_b": null, "target_columns": [ [ 65, 1 ], [ 324, 3 ], [ 344, 5 ], [ 420, 8 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 683, 34 ] ], "id": "L16.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 428, 17 ] ], "source_columns_b": null, "target_columns": [ [ 428, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 717, 34 ] ], "id": "L16.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 49, 1 ], [ 51, 2 ], [ 249, 2 ], [ 252, 12 ] ], "source_columns_b": null, "target_columns": [ [ 49, 1 ], [ 51, 2 ], [ 249, 2 ], [ 252, 12 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 751, 34 ] ], "id": "L16.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 23, 1 ], [ 26, 2 ], [ 66, 1 ], [ 111, 1 ], [ 113, 3 ], [ 117, 1 ], [ 120, 7 ], [ 330, 1 ] ], "source_columns_b": null, "target_columns": [ [ 23, 1 ], [ 26, 2 ], [ 66, 1 ], [ 111, 1 ], [ 113, 3 ], [ 117, 1 ], [ 120, 7 ], [ 330, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 785, 4 ] ], "id": "L16.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 21, 1 ], [ 116, 1 ] ], "source_columns_b": null, "target_columns": [ [ 21, 1 ], [ 116, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:340", "l:313", "l:293", "l:179", "l:178", "l:255", "l:277", "l:262", "l:339", "l:341", "l:351", "l:354", "l:350", "l:256" ] }, { "active_attention_heads": 3, "active_mlp_neurons": 676, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L17.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:78", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 67, 1 ] ], "source_columns_b": null, "target_columns": [ [ 352, 1 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L17.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:261", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 28, 1 ], [ 30, 3 ], [ 153, 13 ], [ 241, 8 ], [ 296, 1 ], [ 331, 8 ], [ 63, 2 ], [ 70, 6 ], [ 315, 2 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 349, 2 ] ], "source_columns_b": null, "target_columns": [ [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 1 ], "id": "L17.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 22, 1 ], [ 47, 1 ], [ 272, 1 ], [ 351, 1 ], [ 353, 16 ] ], "type": "cancel" } ], "emitted_attention_heads": 3, "index": 17, "mlp_operations": [ { "hidden_slots": [ [ 1, 200 ] ], "id": "L17.mlp.0", "node": "l:292", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 445, 34 ], [ 106, 1 ], [ 267, 1 ] ], "source_columns_b": null, "target_columns": [ [ 26, 2 ], [ 47, 1 ], [ 49, 1 ], [ 51, 2 ], [ 66, 1 ], [ 249, 2 ], [ 252, 8 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 201, 62 ] ], "id": "L17.mlp.1", "node": "l:77", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 352, 1 ] ], "source_columns_b": null, "target_columns": [ [ 260, 4 ], [ 272, 13 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 263, 17 ] ], "id": "L17.mlp.2", "node": "l:254", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ], [ 53, 8 ], [ 265, 2 ], [ 269, 3 ], [ 290, 3 ], [ 297, 1 ] ], "source_columns_b": null, "target_columns": [ [ 65, 1 ], [ 285, 5 ], [ 324, 3 ], [ 346, 3 ], [ 351, 1 ], [ 353, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 280, 17 ] ], "id": "L17.mlp.3", "node": "l:338", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 48, 1 ], [ 61, 2 ], [ 298, 9 ], [ 308, 4 ], [ 313, 2 ] ], "source_columns_b": null, "target_columns": [ [ 357, 12 ], [ 420, 5 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 297, 1 ] ], "id": "L17.mlp.4", "node": "l:260", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "source_columns_b": null, "target_columns": [ [ 425, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 298, 34 ] ], "id": "L17.mlp.5", "node": "l:349", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 83, 6 ], [ 90, 5 ], [ 329, 1 ], [ 414, 1 ], [ 416, 4 ], [ 19, 1 ], [ 95, 1 ], [ 97, 1 ], [ 479, 10 ], [ 490, 4 ], [ 386, 17 ] ], "source_columns_b": null, "target_columns": [ [ 426, 17 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 332, 34 ] ], "id": "L17.mlp.6", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 445, 17 ] ], "source_columns_b": null, "target_columns": [ [ 445, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 366, 34 ] ], "id": "L17.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 462, 17 ] ], "source_columns_b": null, "target_columns": [ [ 462, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 400, 34 ] ], "id": "L17.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 61, 2 ], [ 298, 9 ], [ 308, 4 ], [ 313, 2 ] ], "source_columns_b": null, "target_columns": [ [ 61, 2 ], [ 298, 9 ], [ 308, 4 ], [ 313, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 434, 34 ] ], "id": "L17.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 28, 1 ], [ 30, 3 ], [ 153, 13 ] ], "source_columns_b": null, "target_columns": [ [ 28, 1 ], [ 30, 3 ], [ 153, 13 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 468, 34 ] ], "id": "L17.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "source_columns_b": null, "target_columns": [ [ 241, 8 ], [ 296, 1 ], [ 331, 8 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 502, 34 ] ], "id": "L17.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 63, 2 ], [ 70, 6 ], [ 315, 2 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 349, 2 ] ], "source_columns_b": null, "target_columns": [ [ 63, 2 ], [ 70, 6 ], [ 315, 2 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 349, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 536, 34 ] ], "id": "L17.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 83, 6 ], [ 90, 5 ], [ 329, 1 ], [ 414, 1 ], [ 416, 4 ] ], "source_columns_b": null, "target_columns": [ [ 83, 6 ], [ 90, 5 ], [ 329, 1 ], [ 414, 1 ], [ 416, 4 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 570, 34 ] ], "id": "L17.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 19, 1 ], [ 95, 1 ], [ 97, 1 ], [ 479, 10 ], [ 490, 4 ] ], "source_columns_b": null, "target_columns": [ [ 19, 1 ], [ 95, 1 ], [ 97, 1 ], [ 479, 10 ], [ 490, 4 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 604, 34 ] ], "id": "L17.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 386, 17 ] ], "source_columns_b": null, "target_columns": [ [ 386, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 638, 34 ] ], "id": "L17.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 53, 8 ], [ 265, 2 ], [ 269, 3 ], [ 290, 3 ], [ 297, 1 ] ], "source_columns_b": null, "target_columns": [ [ 53, 8 ], [ 265, 2 ], [ 269, 3 ], [ 290, 3 ], [ 297, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 672, 4 ] ], "id": "L17.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 106, 1 ], [ 267, 1 ] ], "source_columns_b": null, "target_columns": [ [ 106, 1 ], [ 267, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:292", "l:78", "l:70", "l:254", "l:249", "l:261", "l:291", "l:338", "l:334", "l:349", "l:260", "l:440", "l:77" ] }, { "active_attention_heads": 4, "active_mlp_neurons": 180, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L18.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:248", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 42, 3 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ], [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ], [ 65, 1 ], [ 285, 5 ], [ 324, 3 ], [ 346, 3 ], [ 351, 1 ], [ 353, 4 ] ], "source_columns_b": null, "target_columns": [ [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L18.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:333", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 369, 17 ], [ 357, 12 ], [ 420, 5 ], [ 68, 1 ], [ 76, 7 ], [ 403, 1 ], [ 405, 8 ] ], "source_columns_b": null, "target_columns": [ [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 2 ], "id": "L18.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 42, 3 ], [ 65, 1 ], [ 67, 1 ], [ 207, 5 ], [ 213, 4 ], [ 218, 4 ], [ 227, 1 ], [ 285, 5 ], [ 324, 3 ], [ 346, 3 ], [ 351, 35 ], [ 420, 5 ] ], "type": "cancel" } ], "emitted_attention_heads": 4, "index": 18, "mlp_operations": [ { "hidden_slots": [ [ 1, 17 ] ], "id": "L18.mlp.0", "node": "l:69", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ], [ 260, 4 ], [ 272, 13 ] ], "source_columns_b": null, "target_columns": [ [ 42, 3 ], [ 213, 1 ], [ 215, 2 ], [ 218, 4 ], [ 227, 1 ], [ 241, 4 ], [ 334, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 18, 17 ] ], "id": "L18.mlp.1", "node": "l:290", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 48, 1 ], [ 26, 2 ], [ 47, 1 ], [ 49, 1 ], [ 51, 2 ], [ 66, 1 ], [ 249, 2 ], [ 252, 8 ] ], "source_columns_b": null, "target_columns": [ [ 53, 3 ], [ 67, 1 ], [ 245, 4 ], [ 265, 3 ], [ 269, 3 ], [ 287, 1 ], [ 336, 1 ], [ 391, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 35, 1 ] ], "id": "L18.mlp.2", "node": "l:247", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ] ], "source_columns_b": null, "target_columns": [ [ 56, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 36, 2 ] ], "id": "L18.mlp.3", "node": "l:259", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 425, 1 ] ], "source_columns_b": null, "target_columns": [ [ 286, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L18.mlp.4", "node": "l:451", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 57, 4 ], [ 285, 1 ], [ 288, 5 ], [ 296, 5 ], [ 337, 1 ], [ 393, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 38, 34 ] ], "id": "L18.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 26, 2 ], [ 47, 1 ], [ 49, 1 ], [ 51, 2 ], [ 66, 1 ], [ 249, 2 ], [ 252, 8 ] ], "source_columns_b": null, "target_columns": [ [ 26, 2 ], [ 47, 1 ], [ 49, 1 ], [ 51, 2 ], [ 66, 1 ], [ 249, 2 ], [ 252, 8 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 72, 34 ] ], "id": "L18.mlp.6", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "source_columns_b": null, "target_columns": [ [ 45, 2 ], [ 224, 3 ], [ 228, 4 ], [ 233, 8 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 106, 34 ] ], "id": "L18.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 68, 1 ], [ 76, 7 ], [ 403, 1 ], [ 405, 8 ] ], "source_columns_b": null, "target_columns": [ [ 68, 1 ], [ 76, 7 ], [ 403, 1 ], [ 405, 8 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 140, 34 ] ], "id": "L18.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 260, 4 ], [ 272, 13 ] ], "source_columns_b": null, "target_columns": [ [ 260, 4 ], [ 272, 13 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 174, 2 ] ], "id": "L18.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 48, 1 ] ], "source_columns_b": null, "target_columns": [ [ 48, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 176, 2 ] ], "id": "L18.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 18, 1 ] ], "source_columns_b": null, "target_columns": [ [ 18, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 178, 2 ] ], "id": "L18.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 425, 1 ] ], "source_columns_b": null, "target_columns": [ [ 425, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:69", "l:33", "l:248", "l:290", "l:286", "l:333", "l:247", "l:259", "l:430", "l:451" ] }, { "active_attention_heads": 4, "active_mlp_neurons": 80, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L19.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:32", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 36, 2 ], [ 172, 6 ], [ 181, 26 ], [ 42, 3 ], [ 213, 1 ], [ 215, 2 ], [ 218, 4 ], [ 227, 1 ], [ 241, 4 ], [ 334, 2 ] ], "source_columns_b": null, "target_columns": [ [ 18, 1 ], [ 26, 2 ], [ 36, 1 ], [ 66, 1 ], [ 76, 2 ], [ 172, 6 ], [ 181, 3 ], [ 187, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L19.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:285", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 513, 17 ], [ 53, 3 ], [ 67, 1 ], [ 245, 4 ], [ 265, 3 ], [ 269, 3 ], [ 287, 1 ], [ 336, 1 ], [ 391, 1 ], [ 33, 3 ], [ 38, 2 ], [ 50, 1 ], [ 166, 1 ], [ 168, 4 ], [ 178, 3 ], [ 251, 1 ], [ 327, 1 ], [ 404, 1 ] ], "source_columns_b": null, "target_columns": [ [ 37, 1 ], [ 184, 3 ], [ 188, 13 ] ], "type": "compute_linear" }, { "heads": [ 2, 2 ], "id": "L19.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 36, 2 ], [ 42, 3 ], [ 172, 6 ], [ 181, 26 ], [ 213, 1 ], [ 215, 2 ], [ 218, 4 ], [ 227, 1 ], [ 241, 4 ], [ 334, 2 ], [ 513, 17 ] ], "type": "cancel" } ], "emitted_attention_heads": 4, "index": 19, "mlp_operations": [ { "hidden_slots": [ [ 1, 1 ] ], "id": "L19.mlp.0", "node": "l:31", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ], [ 26, 2 ], [ 36, 1 ], [ 66, 1 ], [ 76, 2 ], [ 172, 6 ], [ 181, 3 ], [ 187, 1 ] ], "source_columns_b": null, "target_columns": [ [ 201, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 2, 1 ] ], "id": "L19.mlp.1", "node": "l:284", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 37, 1 ], [ 184, 3 ], [ 188, 13 ] ], "source_columns_b": null, "target_columns": [ [ 202, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 1 ] ], "id": "L19.mlp.2", "node": "l:332", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ] ], "source_columns_b": null, "target_columns": [ [ 203, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 4, 2 ] ], "id": "L19.mlp.3", "node": "l:246", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 56, 1 ] ], "source_columns_b": null, "target_columns": [ [ 204, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 6, 2 ] ], "id": "L19.mlp.4", "node": "l:344", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 264, 1 ] ], "source_columns_b": null, "target_columns": [ [ 205, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 8, 34 ] ], "id": "L19.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 53, 3 ], [ 67, 1 ], [ 245, 4 ], [ 265, 3 ], [ 269, 3 ], [ 287, 1 ], [ 336, 1 ], [ 391, 1 ] ], "source_columns_b": null, "target_columns": [ [ 53, 3 ], [ 67, 1 ], [ 245, 4 ], [ 265, 3 ], [ 269, 3 ], [ 287, 1 ], [ 336, 1 ], [ 391, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 42, 34 ] ], "id": "L19.mlp.6", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 33, 3 ], [ 38, 2 ], [ 50, 1 ], [ 166, 1 ], [ 168, 4 ], [ 178, 3 ], [ 251, 1 ], [ 327, 1 ], [ 404, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 3 ], [ 38, 2 ], [ 50, 1 ], [ 166, 1 ], [ 168, 4 ], [ 178, 3 ], [ 251, 1 ], [ 327, 1 ], [ 404, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 76, 2 ] ], "id": "L19.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 264, 1 ] ], "source_columns_b": null, "target_columns": [ [ 264, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 78, 2 ] ], "id": "L19.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 56, 1 ] ], "source_columns_b": null, "target_columns": [ [ 56, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:32", "l:285", "l:31", "l:246", "l:284", "l:332", "l:344", "l:371", "l:392", "l:410", "l:420" ] }, { "active_attention_heads": 2, "active_mlp_neurons": 23, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L20.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:245", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 201, 1 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L20.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:283", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 202, 1 ] ], "source_columns_b": null, "target_columns": [ [ 53, 1 ] ], "type": "compute_linear" } ], "emitted_attention_heads": 2, "index": 20, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L20.mlp.0", "node": "l:243", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 50, 1 ], [ 204, 1 ] ], "source_columns_b": null, "target_columns": [ [ 54, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L20.mlp.1", "node": "l:257", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 50, 1 ], [ 204, 1 ], [ 286, 1 ] ], "source_columns_b": null, "target_columns": [ [ 55, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 4 ] ], "id": "L20.mlp.2", "node": "l:30", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 201, 1 ] ], "source_columns_b": null, "target_columns": [ [ 56, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 9, 2 ] ], "id": "L20.mlp.3", "node": "l:281", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 50, 1 ], [ 204, 1 ], [ 286, 1 ], [ 53, 1 ] ], "source_columns_b": null, "target_columns": [ [ 166, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 11, 2 ] ], "id": "L20.mlp.4", "node": "l:330", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 50, 1 ], [ 204, 1 ], [ 286, 1 ], [ 53, 1 ], [ 203, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L20.mlp.5", "node": "l:245", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L20.mlp.6", "node": "l:283", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 53, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 13, 2 ] ], "id": "L20.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 201, 1 ] ], "source_columns_b": null, "target_columns": [ [ 201, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 15, 2 ] ], "id": "L20.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 204, 1 ] ], "source_columns_b": null, "target_columns": [ [ 204, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 17, 2 ] ], "id": "L20.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 286, 1 ] ], "source_columns_b": null, "target_columns": [ [ 286, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 19, 2 ] ], "id": "L20.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 202, 1 ] ], "source_columns_b": null, "target_columns": [ [ 202, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 21, 2 ] ], "id": "L20.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 203, 1 ] ], "source_columns_b": null, "target_columns": [ [ 203, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:245", "l:283", "l:244", "l:243", "l:258", "l:257", "l:282", "l:281", "l:331", "l:330", "l:30" ] }, { "active_attention_heads": 1, "active_mlp_neurons": 13, "attention_operations": [], "emitted_attention_heads": 0, "index": 21, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L21.mlp.0", "node": "l:280", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 166, 1 ] ], "source_columns_b": null, "target_columns": [ [ 168, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L21.mlp.1", "node": "l:329", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 1 ] ], "source_columns_b": null, "target_columns": [ [ 169, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 2 ] ], "id": "L21.mlp.2", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 50, 1 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 7, 2 ] ], "id": "L21.mlp.3", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 53, 1 ] ], "source_columns_b": null, "target_columns": [ [ 53, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 9, 2 ] ], "id": "L21.mlp.4", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 166, 1 ] ], "source_columns_b": null, "target_columns": [ [ 166, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 11, 2 ] ], "id": "L21.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 33, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:280", "l:329", "l:29" ] }, { "active_attention_heads": 2, "active_mlp_neurons": 5, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L22.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:28", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 56, 1 ], [ 54, 2 ], [ 168, 2 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L22.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 54, 3 ], [ 168, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 2, "index": 22, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L22.mlp.0", "node": "l:27", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 205, 1 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "add_into_bypass" }, { "hidden_slots": [], "id": "L22.mlp.1", "node": "l:28", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L22.mlp.2", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 169, 1 ] ], "source_columns_b": null, "target_columns": [ [ 169, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:28", "l:27" ] }, { "active_attention_heads": 1, "active_mlp_neurons": 47, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L23.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 205, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 1, "index": 23, "mlp_operations": [ { "hidden_slots": [ [ 1, 46 ] ], "id": "L23.mlp.0", "node": "l:26", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 50, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 3 ], [ 53, 4 ], [ 166, 1 ], [ 168, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L23.mlp.1", "node": "l:361", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 38, 2 ], [ 42, 1 ], [ 178, 3 ], [ 201, 6 ], [ 213, 1 ], [ 215, 1 ], [ 327, 1 ], [ 334, 1 ], [ 404, 1 ] ], "type": "compute_literal_value" } ], "newly_computed": [ "l:361", "l:348", "l:25", "l:370", "l:391", "l:409", "l:419", "l:429", "l:439", "l:449", "l:26" ] }, { "active_attention_heads": 1, "active_mlp_neurons": 1963, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L24.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 1, "index": 24, "mlp_operations": [ { "hidden_slots": [ [ 1, 204 ] ], "id": "L24.mlp.0", "node": "l:24", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 3 ], [ 53, 4 ], [ 166, 1 ], [ 168, 4 ], [ 18, 1 ], [ 26, 2 ], [ 36, 1 ], [ 66, 1 ], [ 76, 2 ], [ 172, 6 ], [ 181, 3 ], [ 187, 1 ], [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ], [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ], [ 37, 1 ], [ 184, 3 ], [ 188, 13 ], [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ], [ 426, 17 ], [ 38, 2 ], [ 42, 1 ], [ 178, 3 ], [ 201, 6 ], [ 213, 1 ], [ 215, 1 ], [ 327, 1 ], [ 334, 1 ], [ 404, 1 ], [ 38, 2 ], [ 42, 1 ], [ 178, 3 ], [ 201, 6 ], [ 213, 1 ], [ 215, 1 ], [ 327, 1 ], [ 334, 1 ], [ 404, 1 ], [ 38, 2 ], [ 42, 1 ], [ 178, 3 ], [ 201, 6 ], [ 213, 1 ], [ 215, 1 ], [ 327, 1 ], [ 334, 1 ], [ 404, 1 ], [ 38, 2 ], [ 42, 1 ], [ 178, 3 ], [ 201, 6 ], [ 213, 1 ], [ 215, 1 ], [ 327, 1 ], [ 334, 1 ], [ 404, 1 ], [ 38, 2 ], [ 42, 1 ], [ 178, 3 ], [ 201, 6 ], [ 213, 1 ], [ 215, 1 ], [ 327, 1 ], [ 334, 1 ], [ 404, 1 ], [ 38, 2 ], [ 42, 1 ], [ 178, 3 ], [ 201, 6 ], [ 213, 1 ], [ 215, 1 ], [ 327, 1 ], [ 334, 1 ], [ 404, 1 ] ], "source_columns_b": null, "target_columns": [ [ 43, 10 ], [ 61, 5 ], [ 67, 2 ], [ 70, 6 ], [ 78, 11 ], [ 216, 1 ], [ 218, 4 ], [ 224, 8 ], [ 233, 4 ], [ 238, 4 ], [ 243, 4 ], [ 248, 3 ], [ 252, 10 ], [ 263, 1 ], [ 265, 3 ], [ 269, 16 ], [ 286, 2 ], [ 301, 6 ], [ 308, 4 ], [ 313, 4 ], [ 318, 1 ], [ 320, 3 ], [ 324, 3 ], [ 328, 2 ], [ 332, 1 ], [ 335, 2 ], [ 338, 1 ], [ 346, 12 ], [ 359, 4 ], [ 364, 3 ], [ 369, 9 ], [ 379, 13 ], [ 394, 9 ], [ 405, 7 ], [ 414, 1 ], [ 416, 10 ], [ 443, 15 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 205, 204 ] ], "id": "L24.mlp.1", "node": "l:369", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 3 ], [ 53, 4 ], [ 166, 1 ], [ 168, 4 ], [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ], [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ], [ 37, 1 ], [ 184, 3 ], [ 188, 13 ], [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ], [ 426, 17 ], [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ], [ 18, 1 ], [ 26, 2 ], [ 36, 1 ], [ 66, 1 ], [ 76, 2 ], [ 172, 6 ], [ 181, 3 ], [ 187, 1 ], [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ], [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ], [ 37, 1 ], [ 184, 3 ], [ 188, 13 ], [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ], [ 426, 17 ] ], "source_columns_b": null, "target_columns": [ [ 20, 1 ], [ 69, 1 ], [ 97, 4 ], [ 102, 4 ], [ 344, 2 ], [ 458, 6 ], [ 465, 14 ], [ 480, 9 ], [ 490, 4 ], [ 495, 4 ], [ 500, 4 ], [ 505, 1 ], [ 507, 6 ], [ 515, 4 ], [ 520, 1 ], [ 530, 1 ], [ 533, 138 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 409, 204 ] ], "id": "L24.mlp.2", "node": "l:390", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 3 ], [ 53, 4 ], [ 166, 1 ], [ 168, 4 ], [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ], [ 37, 1 ], [ 184, 3 ], [ 188, 13 ], [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ], [ 426, 17 ], [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 26 ], [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ], [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ], [ 37, 1 ], [ 184, 3 ], [ 188, 13 ], [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ], [ 426, 17 ], [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "source_columns_b": null, "target_columns": [ [ 671, 204 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 613, 204 ] ], "id": "L24.mlp.3", "node": "l:408", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 3 ], [ 53, 4 ], [ 166, 1 ], [ 168, 4 ], [ 37, 1 ], [ 184, 3 ], [ 188, 13 ], [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ], [ 426, 17 ], [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 26 ], [ 136, 17 ], [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ], [ 37, 1 ], [ 184, 3 ], [ 188, 13 ], [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ], [ 426, 17 ], [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 26 ] ], "source_columns_b": null, "target_columns": [ [ 875, 204 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 817, 204 ] ], "id": "L24.mlp.4", "node": "l:418", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 3 ], [ 53, 4 ], [ 166, 1 ], [ 168, 4 ], [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ], [ 426, 17 ], [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 26 ], [ 136, 17 ], [ 136, 17 ], [ 37, 1 ], [ 184, 3 ], [ 188, 13 ], [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ], [ 426, 17 ], [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 26 ], [ 136, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1079, 204 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1021, 204 ] ], "id": "L24.mlp.5", "node": "l:428", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 3 ], [ 53, 4 ], [ 166, 1 ], [ 168, 4 ], [ 426, 17 ], [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 26 ], [ 136, 17 ], [ 136, 17 ], [ 136, 17 ], [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ], [ 426, 17 ], [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 26 ], [ 136, 17 ], [ 136, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1283, 204 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1225, 204 ] ], "id": "L24.mlp.6", "node": "l:438", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 3 ], [ 53, 4 ], [ 166, 1 ], [ 168, 4 ], [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 26 ], [ 136, 17 ], [ 136, 17 ], [ 136, 17 ], [ 136, 17 ], [ 426, 17 ], [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 26 ], [ 136, 17 ], [ 136, 17 ], [ 136, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1487, 204 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1429, 204 ] ], "id": "L24.mlp.7", "node": "l:448", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 3 ], [ 53, 4 ], [ 166, 1 ], [ 168, 4 ], [ 136, 17 ], [ 136, 17 ], [ 136, 17 ], [ 136, 17 ], [ 136, 17 ], [ 136, 17 ], [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 26 ], [ 136, 17 ], [ 136, 17 ], [ 136, 17 ], [ 136, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1691, 204 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1633, 34 ] ], "id": "L24.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 18, 1 ], [ 26, 2 ], [ 36, 1 ], [ 66, 1 ], [ 76, 2 ], [ 172, 6 ], [ 181, 3 ], [ 187, 1 ] ], "source_columns_b": null, "target_columns": [ [ 18, 1 ], [ 26, 2 ], [ 36, 1 ], [ 66, 1 ], [ 76, 2 ], [ 172, 6 ], [ 181, 3 ], [ 187, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1667, 34 ] ], "id": "L24.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ] ], "source_columns_b": null, "target_columns": [ [ 19, 1 ], [ 28, 1 ], [ 30, 2 ], [ 90, 6 ], [ 106, 1 ], [ 153, 5 ], [ 331, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1701, 34 ] ], "id": "L24.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "source_columns_b": null, "target_columns": [ [ 21, 3 ], [ 111, 1 ], [ 113, 5 ], [ 120, 7 ], [ 330, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1735, 34 ] ], "id": "L24.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 37, 1 ], [ 184, 3 ], [ 188, 13 ] ], "source_columns_b": null, "target_columns": [ [ 37, 1 ], [ 184, 3 ], [ 188, 13 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1769, 34 ] ], "id": "L24.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ] ], "source_columns_b": null, "target_columns": [ [ 32, 1 ], [ 158, 8 ], [ 207, 5 ], [ 214, 1 ], [ 333, 1 ], [ 392, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1803, 34 ] ], "id": "L24.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 426, 17 ] ], "source_columns_b": null, "target_columns": [ [ 426, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1837, 34 ] ], "id": "L24.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "source_columns_b": null, "target_columns": [ [ 24, 2 ], [ 40, 2 ], [ 107, 2 ], [ 110, 1 ], [ 112, 1 ], [ 127, 9 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1871, 34 ] ], "id": "L24.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 136, 17 ] ], "source_columns_b": null, "target_columns": [ [ 136, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1905, 34 ] ], "id": "L24.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 38, 2 ], [ 42, 1 ], [ 178, 3 ], [ 201, 6 ], [ 213, 1 ], [ 215, 1 ], [ 327, 1 ], [ 334, 1 ], [ 404, 1 ] ], "source_columns_b": null, "target_columns": [ [ 38, 2 ], [ 42, 1 ], [ 178, 3 ], [ 201, 6 ], [ 213, 1 ], [ 215, 1 ], [ 327, 1 ], [ 334, 1 ], [ 404, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1939, 24 ] ], "id": "L24.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 33, 3 ], [ 53, 4 ], [ 166, 1 ], [ 168, 4 ] ], "source_columns_b": null, "target_columns": [ [ 33, 3 ], [ 53, 4 ], [ 166, 1 ], [ 168, 4 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:24", "l:369", "l:390", "l:408", "l:418", "l:428", "l:438", "l:448" ] }, { "active_attention_heads": 32, "active_mlp_neurons": 3415, "attention_operations": [ { "heads": [ 0, 4 ], "id": "L25.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:23", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 43, 10 ], [ 61, 5 ], [ 67, 2 ], [ 70, 6 ], [ 78, 11 ], [ 216, 1 ], [ 218, 4 ], [ 224, 8 ], [ 233, 4 ], [ 238, 4 ], [ 243, 4 ], [ 248, 3 ], [ 252, 10 ], [ 263, 1 ], [ 265, 3 ], [ 269, 16 ], [ 286, 2 ], [ 301, 6 ], [ 308, 4 ], [ 313, 4 ], [ 318, 1 ], [ 320, 3 ], [ 324, 3 ], [ 328, 2 ], [ 332, 1 ], [ 335, 2 ], [ 338, 1 ], [ 346, 12 ], [ 359, 4 ], [ 364, 3 ], [ 369, 9 ], [ 379, 13 ], [ 394, 9 ], [ 405, 7 ], [ 414, 1 ], [ 416, 10 ], [ 443, 15 ] ], "source_columns_b": null, "target_columns": [ [ 426, 17 ] ], "type": "compute_linear" }, { "heads": [ 4, 4 ], "id": "L25.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:368", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 20, 1 ], [ 69, 1 ], [ 97, 4 ], [ 102, 4 ], [ 344, 2 ], [ 458, 6 ], [ 465, 14 ], [ 480, 9 ], [ 490, 4 ], [ 495, 4 ], [ 500, 4 ], [ 505, 1 ], [ 507, 6 ], [ 515, 4 ], [ 520, 1 ], [ 530, 1 ], [ 533, 138 ] ], "source_columns_b": null, "target_columns": [ [ 18, 2 ], [ 21, 2 ], [ 90, 6 ], [ 106, 3 ], [ 110, 4 ] ], "type": "compute_linear" }, { "heads": [ 8, 4 ], "id": "L25.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:389", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 671, 204 ] ], "source_columns_b": null, "target_columns": [ [ 23, 3 ], [ 114, 4 ], [ 120, 9 ], [ 330, 1 ] ], "type": "compute_linear" }, { "heads": [ 12, 4 ], "id": "L25.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": "l:407", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 875, 204 ] ], "source_columns_b": null, "target_columns": [ [ 26, 3 ], [ 66, 1 ], [ 129, 12 ], [ 331, 1 ] ], "type": "compute_linear" }, { "heads": [ 16, 4 ], "id": "L25.attn.4", "key_source_columns": [ [ 0, 1 ] ], "node": "l:417", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1079, 204 ] ], "source_columns_b": null, "target_columns": [ [ 30, 2 ], [ 141, 15 ] ], "type": "compute_linear" }, { "heads": [ 20, 4 ], "id": "L25.attn.5", "key_source_columns": [ [ 0, 1 ] ], "node": "l:427", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1283, 204 ] ], "source_columns_b": null, "target_columns": [ [ 32, 2 ], [ 156, 11 ], [ 168, 4 ] ], "type": "compute_linear" }, { "heads": [ 24, 4 ], "id": "L25.attn.6", "key_source_columns": [ [ 0, 1 ] ], "node": "l:437", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1487, 204 ] ], "source_columns_b": null, "target_columns": [ [ 34, 3 ], [ 172, 13 ], [ 392, 1 ] ], "type": "compute_linear" }, { "heads": [ 28, 4 ], "id": "L25.attn.7", "key_source_columns": [ [ 0, 1 ] ], "node": "l:447", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1691, 204 ] ], "source_columns_b": null, "target_columns": [ [ 37, 3 ], [ 185, 12 ], [ 327, 1 ], [ 333, 1 ] ], "type": "compute_linear" } ], "emitted_attention_heads": 32, "index": 25, "mlp_operations": [ { "hidden_slots": [ [ 1, 17 ] ], "id": "L25.mlp.0", "node": "l:12", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 339, 1 ], [ 426, 17 ] ], "source_columns_b": null, "target_columns": [ [ 40, 2 ], [ 197, 14 ], [ 404, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 18, 17 ] ], "id": "L25.mlp.1", "node": "l:362", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 340, 1 ], [ 18, 2 ], [ 21, 2 ], [ 90, 6 ], [ 106, 3 ], [ 110, 4 ] ], "source_columns_b": null, "target_columns": [ [ 42, 1 ], [ 53, 4 ], [ 76, 2 ], [ 211, 1 ], [ 213, 3 ], [ 334, 1 ], [ 1895, 5 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 35, 17 ] ], "id": "L25.mlp.2", "node": "l:383", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 293, 1 ], [ 23, 3 ], [ 114, 4 ], [ 120, 9 ], [ 330, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1900, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 52, 17 ] ], "id": "L25.mlp.3", "node": "l:401", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 341, 1 ], [ 26, 3 ], [ 66, 1 ], [ 129, 12 ], [ 331, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1917, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 69, 17 ] ], "id": "L25.mlp.4", "node": "l:411", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 294, 1 ], [ 30, 2 ], [ 141, 15 ] ], "source_columns_b": null, "target_columns": [ [ 1934, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 86, 17 ] ], "id": "L25.mlp.5", "node": "l:421", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 342, 1 ], [ 32, 2 ], [ 156, 11 ], [ 168, 4 ] ], "source_columns_b": null, "target_columns": [ [ 1951, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 103, 17 ] ], "id": "L25.mlp.6", "node": "l:431", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 295, 1 ], [ 34, 3 ], [ 172, 13 ], [ 392, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1968, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 120, 17 ] ], "id": "L25.mlp.7", "node": "l:441", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 343, 1 ], [ 37, 3 ], [ 185, 12 ], [ 327, 1 ], [ 333, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1985, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 137, 408 ] ], "id": "L25.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 43, 10 ], [ 61, 5 ], [ 67, 2 ], [ 70, 6 ], [ 78, 11 ], [ 216, 1 ], [ 218, 4 ], [ 224, 8 ], [ 233, 4 ], [ 238, 4 ], [ 243, 4 ], [ 248, 3 ], [ 252, 10 ], [ 263, 1 ], [ 265, 3 ], [ 269, 16 ], [ 286, 2 ], [ 301, 6 ], [ 308, 4 ], [ 313, 4 ], [ 318, 1 ], [ 320, 3 ], [ 324, 3 ], [ 328, 2 ], [ 332, 1 ], [ 335, 2 ], [ 338, 1 ], [ 346, 12 ], [ 359, 4 ], [ 364, 3 ], [ 369, 9 ], [ 379, 13 ], [ 394, 9 ], [ 405, 7 ], [ 414, 1 ], [ 416, 10 ], [ 443, 15 ] ], "source_columns_b": null, "target_columns": [ [ 43, 10 ], [ 61, 5 ], [ 67, 2 ], [ 70, 6 ], [ 78, 11 ], [ 216, 1 ], [ 218, 4 ], [ 224, 8 ], [ 233, 4 ], [ 238, 4 ], [ 243, 4 ], [ 248, 3 ], [ 252, 10 ], [ 263, 1 ], [ 265, 3 ], [ 269, 16 ], [ 286, 2 ], [ 301, 6 ], [ 308, 4 ], [ 313, 4 ], [ 318, 1 ], [ 320, 3 ], [ 324, 3 ], [ 328, 2 ], [ 332, 1 ], [ 335, 2 ], [ 338, 1 ], [ 346, 12 ], [ 359, 4 ], [ 364, 3 ], [ 369, 9 ], [ 379, 13 ], [ 394, 9 ], [ 405, 7 ], [ 414, 1 ], [ 416, 10 ], [ 443, 15 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 545, 408 ] ], "id": "L25.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 20, 1 ], [ 69, 1 ], [ 97, 4 ], [ 102, 4 ], [ 344, 2 ], [ 458, 6 ], [ 465, 14 ], [ 480, 9 ], [ 490, 4 ], [ 495, 4 ], [ 500, 4 ], [ 505, 1 ], [ 507, 6 ], [ 515, 4 ], [ 520, 1 ], [ 530, 1 ], [ 533, 138 ] ], "source_columns_b": null, "target_columns": [ [ 20, 1 ], [ 69, 1 ], [ 97, 4 ], [ 102, 4 ], [ 344, 2 ], [ 458, 6 ], [ 465, 14 ], [ 480, 9 ], [ 490, 4 ], [ 495, 4 ], [ 500, 4 ], [ 505, 1 ], [ 507, 6 ], [ 515, 4 ], [ 520, 1 ], [ 530, 1 ], [ 533, 138 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 953, 408 ] ], "id": "L25.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 671, 204 ] ], "source_columns_b": null, "target_columns": [ [ 671, 204 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1361, 408 ] ], "id": "L25.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 875, 204 ] ], "source_columns_b": null, "target_columns": [ [ 875, 204 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1769, 408 ] ], "id": "L25.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1079, 204 ] ], "source_columns_b": null, "target_columns": [ [ 1079, 204 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2177, 408 ] ], "id": "L25.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1283, 204 ] ], "source_columns_b": null, "target_columns": [ [ 1283, 204 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2585, 408 ] ], "id": "L25.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1487, 204 ] ], "source_columns_b": null, "target_columns": [ [ 1487, 204 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2993, 408 ] ], "id": "L25.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1691, 204 ] ], "source_columns_b": null, "target_columns": [ [ 1691, 204 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3401, 2 ] ], "id": "L25.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 339, 1 ] ], "source_columns_b": null, "target_columns": [ [ 339, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3403, 2 ] ], "id": "L25.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 340, 1 ] ], "source_columns_b": null, "target_columns": [ [ 340, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3405, 2 ] ], "id": "L25.mlp.18", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 293, 1 ] ], "source_columns_b": null, "target_columns": [ [ 293, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3407, 2 ] ], "id": "L25.mlp.19", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 294, 1 ] ], "source_columns_b": null, "target_columns": [ [ 294, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3409, 2 ] ], "id": "L25.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 342, 1 ] ], "source_columns_b": null, "target_columns": [ [ 342, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3411, 2 ] ], "id": "L25.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 295, 1 ] ], "source_columns_b": null, "target_columns": [ [ 295, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3413, 2 ] ], "id": "L25.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 343, 1 ] ], "source_columns_b": null, "target_columns": [ [ 343, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:23", "l:368", "l:389", "l:407", "l:417", "l:427", "l:437", "l:447", "l:13", "l:12", "l:363", "l:362", "l:384", "l:383", "l:402", "l:401", "l:412", "l:411", "l:422", "l:421", "l:432", "l:431", "l:442", "l:441", "l:11" ] }, { "active_attention_heads": 5, "active_mlp_neurons": 479, "attention_operations": [ { "heads": [ 0, 3 ], "id": "L26.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:10", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 40, 2 ], [ 197, 14 ], [ 404, 1 ], [ 42, 1 ], [ 53, 4 ], [ 76, 2 ], [ 211, 1 ], [ 213, 3 ], [ 334, 1 ], [ 1895, 107 ] ], "source_columns_b": null, "target_columns": [ [ 30, 1 ], [ 141, 1 ], [ 145, 1 ], [ 1631, 3 ], [ 1646, 3 ], [ 1650, 7 ], [ 1694, 1 ] ], "type": "compute_linear" }, { "heads": [ 3, 2 ], "id": "L26.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 30, 4 ], [ 37, 3 ], [ 42, 1 ], [ 53, 4 ], [ 76, 2 ], [ 141, 26 ], [ 168, 4 ], [ 185, 12 ], [ 211, 1 ], [ 213, 3 ], [ 327, 1 ], [ 333, 2 ], [ 341, 1 ], [ 1895, 5 ] ], "type": "cancel" } ], "emitted_attention_heads": 5, "index": 26, "mlp_operations": [ { "hidden_slots": [ [ 1, 34 ] ], "id": "L26.mlp.0", "node": "l:0", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 29, 2 ], [ 141, 1 ], [ 145, 1 ], [ 1631, 3 ], [ 1646, 3 ], [ 1650, 7 ], [ 1694, 1 ], [ 57, 4 ], [ 285, 1 ], [ 288, 5 ], [ 296, 5 ], [ 337, 1 ], [ 393, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 35, 34 ] ], "id": "L26.mlp.1", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 426, 17 ] ], "source_columns_b": null, "target_columns": [ [ 426, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 69, 34 ] ], "id": "L26.mlp.2", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 18, 2 ], [ 21, 2 ], [ 90, 6 ], [ 106, 3 ], [ 110, 4 ] ], "source_columns_b": null, "target_columns": [ [ 18, 2 ], [ 21, 2 ], [ 90, 6 ], [ 106, 3 ], [ 110, 4 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 103, 34 ] ], "id": "L26.mlp.3", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 23, 3 ], [ 114, 4 ], [ 120, 9 ], [ 330, 1 ] ], "source_columns_b": null, "target_columns": [ [ 23, 3 ], [ 114, 4 ], [ 120, 9 ], [ 330, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 137, 34 ] ], "id": "L26.mlp.4", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 26, 3 ], [ 66, 1 ], [ 129, 12 ], [ 331, 1 ] ], "source_columns_b": null, "target_columns": [ [ 26, 3 ], [ 66, 1 ], [ 129, 12 ], [ 331, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 171, 34 ] ], "id": "L26.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 34, 3 ], [ 172, 13 ], [ 392, 1 ] ], "source_columns_b": null, "target_columns": [ [ 34, 3 ], [ 172, 13 ], [ 392, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 205, 34 ] ], "id": "L26.mlp.6", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 40, 2 ], [ 197, 14 ], [ 404, 1 ] ], "source_columns_b": null, "target_columns": [ [ 40, 2 ], [ 197, 14 ], [ 404, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 239, 34 ] ], "id": "L26.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1900, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1900, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 273, 34 ] ], "id": "L26.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1917, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1917, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 307, 34 ] ], "id": "L26.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1934, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1934, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 341, 34 ] ], "id": "L26.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1951, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1951, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 375, 34 ] ], "id": "L26.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1968, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1968, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 409, 34 ] ], "id": "L26.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1985, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1985, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 443, 34 ] ], "id": "L26.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 57, 4 ], [ 285, 1 ], [ 288, 5 ], [ 296, 5 ], [ 337, 1 ], [ 393, 1 ] ], "source_columns_b": null, "target_columns": [ [ 57, 4 ], [ 285, 1 ], [ 288, 5 ], [ 296, 5 ], [ 337, 1 ], [ 393, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 477, 2 ] ], "id": "L26.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 29, 1 ] ], "source_columns_b": null, "target_columns": [ [ 29, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [], "id": "L26.mlp.15", "node": "i:0", "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 0, 1 ] ], "type": "clear_literal_seed" } ], "newly_computed": [ "l:10", "l:1", "l:0" ] } ], "realizations": [ { "candidates": [ "mlp_composite" ], "node": "l:46", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:48", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:50", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:45", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:47", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:49", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:43", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:7", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:54", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:55", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:57", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:52", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:51", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:41", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:38", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:39", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:58", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:36", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:64", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:65", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:67", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:62", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:73", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:74", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:76", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:71", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:105", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:118", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:119", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:121", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:116", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:114", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:115", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:112", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:165", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:163", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:102", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:103", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:106", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:100", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:107", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:99", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:98", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:97", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:96", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:125", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:126", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:123", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:122", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:110", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:127", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:130", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:128", "selected": "mlp_composite" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:108", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:133", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:131", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:154", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:155", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:157", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:152", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:158", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:151", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:150", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:149", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:148", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:169", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:170", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:167", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:166", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:161", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:171", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:174", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:172", "selected": "mlp_composite" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:159", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:177", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:175", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:92", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:93", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:95", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:90", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:144", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:145", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:147", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:142", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:88", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:84", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:85", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:87", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:82", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:200", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:196", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:197", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:199", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:194", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:226", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:222", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:223", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:225", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:220", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:140", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:136", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:137", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:139", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:134", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:190", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:186", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:187", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:189", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:184", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:214", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:210", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:211", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:213", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:208", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:267", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:274", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:353", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:272", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:337", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:270", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:289", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:268", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:265", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:59", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:376", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:300", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:304", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:308", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:299", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:302", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:303", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:306", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:307", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:310", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:297", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:296", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:294", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:315", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:321", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:311", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:317", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:323", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:325", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:356", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:319", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:340", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:313", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:292", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:68", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:379", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:80", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:182", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:206", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:192", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:216", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:232", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:218", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:234", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:240", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:360", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:238", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:237", "selected": "attn_add" }, { "candidates": [ "mlp_composite" ], "node": "l:236", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:230", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:229", "selected": "attn_add" }, { "candidates": [ "mlp_composite" ], "node": "l:228", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:204", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:203", "selected": "attn_add" }, { "candidates": [ "mlp_composite" ], "node": "l:202", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:180", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:179", "selected": "attn_add" }, { "candidates": [ "mlp_composite" ], "node": "l:178", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:79", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:78", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:382", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:34", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:60", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:69", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:32", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:250", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:252", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:254", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:248", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:263", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:275", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:277", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:261", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:287", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:290", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:326", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:285", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:335", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:338", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:341", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:333", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:351", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:354", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:357", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:349", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:374", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:377", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:380", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:372", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:395", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:397", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:399", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:393", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:347", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:345", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:31", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:245", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:247", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:246", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:260", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:259", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:284", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:283", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:332", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:243", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:257", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:281", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:280", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:330", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:329", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:28", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:344", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:27", "selected": "mlp_add" }, { "candidates": [ "mlp_literal" ], "node": "l:361", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:24", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:23", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:369", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:368", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:390", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:389", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:408", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:407", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:418", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:417", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:428", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:427", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:438", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:437", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:448", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:447", "selected": "attn_transport" }, { "candidates": [ "mlp_literal" ], "node": "l:4", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:5", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:9", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:2", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:21", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:20", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:19", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:18", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:17", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:16", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:22", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:14", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:12", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:366", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:367", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:364", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:362", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:387", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:388", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:385", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:383", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:405", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:406", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:403", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:401", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:415", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:416", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:413", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:411", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:425", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:426", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:423", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:421", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:435", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:436", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:433", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:431", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:445", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:446", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:443", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:441", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:10", "selected": "attn_transport" }, { "candidates": [ "mlp_literal" ], "node": "l:451", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:0", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:215", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:191", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:141", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:227", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:201", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:89", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:242", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:359", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:343", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:328", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:279", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:256", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:77", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:30", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:26", "selected": "mlp_composite" } ] }, "token_io": { "bos_token_id": 15, "embedding_residual_columns": [ [ 1, 17 ] ], "eos_token_id": 16, "output_residual_columns": [ [ 1, 17 ] ], "readout": { "checkpoint_tensor": "model.embed_tokens.weight", "final_normalization": "rms_norm", "formula": "logits[token] = final_normalize(residual) @ embedding[token]", "input_state": "output", "kind": "tied_embedding_dot_product" }, "source_embedding_node": "s:8", "unknown_token_id": 17, "vocabulary_sha256": "907a098ae3a8687a22cd33300ec72d9bb9d997acddefeec8d226522dc818ee2c", "vocabulary_size": 18 } }