{ "$schema": "./torchwright_schematic_v1.schema.json", "artifact": { "architecture": "phi3", "files": { "config.json": { "bytes": 999, "sha256": "a055e8431c7a23fc9e4810d153ff5b8e59343a44a1746270305917f372b779d1" }, "generation_config.json": { "bytes": 105, "sha256": "6ebdfc7ea4a3b172e5baf014667c957a7eb90ea465d05178b801cc2ab7b6bdc4" }, "model-00001-of-00019.safetensors": { "bytes": 113533384, "sha256": "36ccb842e49a08c21445dae0b618794f407ce22575bf2ff67471c790a99018ec" }, "model-00002-of-00019.safetensors": { "bytes": 113533384, "sha256": "3f4b71657e58d4e6b972becbb37530d69c903fa5ca571712ad8b3bac8ebbd550" }, "model-00003-of-00019.safetensors": { "bytes": 113533384, "sha256": "54c334b474ca457cecac2dc91b1c122d4727ad6f122dc2ca4dcb9c15bb7d3abf" }, "model-00004-of-00019.safetensors": { "bytes": 113533384, "sha256": "0a4133d46642507acb8a2410afa7ef84d21961376ac453dcd8b69a081f67a1e0" }, "model-00005-of-00019.safetensors": { "bytes": 113533384, "sha256": "9d1fc01a8a565f15db75c247d5ff879ea9359a413381c9002dd54339c10d34e4" }, "model-00006-of-00019.safetensors": { "bytes": 113533384, "sha256": "86dac3d9e353d691e44f2ded89a3a58dc60df23a67b3c3a2a1e0eacf23cf4825" }, "model-00007-of-00019.safetensors": { "bytes": 113533384, "sha256": "1be301d7f7927116acda3e7e656c5c2f4f9b65167a89757c29d54988a1d3d865" }, "model-00008-of-00019.safetensors": { "bytes": 113533384, "sha256": "726974bed3125fa31ef92bd63d09c756e80c32a1d5674565ee79a709df53e708" }, "model-00009-of-00019.safetensors": { "bytes": 113533384, "sha256": "60c8e99f08f7f41a965a362e9af223f4cf8359b9552b58b2f2dd7137a05ac861" }, "model-00010-of-00019.safetensors": { "bytes": 113533384, "sha256": "2d18f3c7a4b0dea53fe646299d79949e1ed178cec053ac25aab9c161f11a958c" }, "model-00011-of-00019.safetensors": { "bytes": 113533384, "sha256": "616f0ed2cbc8e36d20f0044c26167cc629d29627fdd304b9dce05a0a938dc8ff" }, "model-00012-of-00019.safetensors": { "bytes": 113533384, "sha256": "4602a2feb3c0e967e76221aa0e35ed42f36ba597e36419006bd788ea836da41f" }, "model-00013-of-00019.safetensors": { "bytes": 113533384, "sha256": "9bd246c635df773e6baaeb481e94de2c747d4474acaacf6947eb52ea11d1695f" }, "model-00014-of-00019.safetensors": { "bytes": 113533384, "sha256": "1928da49ed77e9facdcfaead869dcf2dd60c4cfb66c43b76ba256620bc3961f8" }, "model-00015-of-00019.safetensors": { "bytes": 113533384, "sha256": "c9fa31c7a026c53a6ef03bee64d1d36ce9d28b3c6d30957ca1784b9953469961" }, "model-00016-of-00019.safetensors": { "bytes": 113533384, "sha256": "ef5963bd73f980d69187a6561a60541bca0bc1be5f24d0b540d3f092679c4cf6" }, "model-00017-of-00019.safetensors": { "bytes": 113533384, "sha256": "4e50e4cef0311347743c43b33a860ce0313f79986a974fa1380ef041ff7ea276" }, "model-00018-of-00019.safetensors": { "bytes": 113533384, "sha256": "21f1377d91fdffe4d67b221e1f759805d20930734c9b086ee9bbc1262656b898" }, "model-00019-of-00019.safetensors": { "bytes": 585664, "sha256": "7007bb14771230ab3129388319b7d18a4b544ec0900c064f72d0a9cba5d7c457" }, "model.safetensors.index.json": { "bytes": 8758, "sha256": "118856a7d5346813dfe3851432b3230475862ed665f63adfd9bf65de2d958895" }, "tokenizer.json": { "bytes": 2339, "sha256": "05336a3eb8133e6b075e6a4b62b18ea6f95350a1b4ea80b90166ea1f8947b2ae" }, "tokenizer_config.json": { "bytes": 199, "sha256": "6f9df026786c39bfda75becfa45ea655ccad3ee7bf4c4e186777e11aa930758b" }, "torchwright_schematic_support.npz": { "bytes": 629805, "sha256": "0c7a1c7db3a422246f76b1d505d9eeab25934f7fe1ff1d2b02d8ec3835fa64c8" }, "torchwright_schematic_v1.schema.json": { "bytes": 3678, "sha256": "1881c7400c00ee66e6e911c7416ceb573b31129cf24bb80d4055b9e2c647ec7f" } }, "kind": "huggingface_causal_lm_bundle" }, "build": { "compiler_code_sha256": "639e918d8b7011d6f247de3e5696cc4faaa3b154fe081a266f956ddaa9e68e1c", "options": { "activation": "swish", "bias": false, "collapse_piecewise_linear": true, "collapse_univariate": true, "d_head": 64, "d_hidden_capacity": 4096, "d_model": 2048, "n_heads_capacity": 32, "optimize": 2, "trim_heads": true }, "schedule_fingerprint": "9fb5f7cf0453a39bf3685a4998928fe20be118a0b6b969bbfa58bb507be2cd84", "schedule_provenance": { "delivery": "fresh", "optimize": 2, "selected_is_optimal": true, "selected_objective": 18, "selected_objective_blocks": [ 18, 0 ], "selected_origin": "solver", "solver_best_bound": 18.0, "solver_is_optimal": true, "solver_objective": 18, "solver_status": "OPTIMAL" }, "torchwright_version": "unknown" }, "format": "torchwright.schematic.v1", "graphs": { "internal_nodes": [ { "id": "i:0", "name": "rope_self_match_const_one", "op": "LiteralValue", "purpose": "compiler_internal", "width": 1 } ], "lowered": { "nodes": [ { "annotation": null, "id": "l:0", "inputs": [ "l:1" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:0" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=33.40152674999999))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.40152674999999, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.40152674999999, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1", "inputs": [ "l:2", "l:10", "l:1057" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 33.40152674999999, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:2", "inputs": [ "l:3", "l:6", "l:7" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:2" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:3", "inputs": [ "l:4", "l:5" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:3" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:4", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:4" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:5", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:5" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:6", "inputs": [ "l:7", "l:9" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:6" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:7", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "fe8947e9c75de0c7a722d4a0a01bc7acb10be2bfac772df3c42693c0f839bc75", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:7" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:8", "inputs": [], "name": null, "op": "Embedding", "parameters": [ { "dtype": "float32", "name": "table", "sha256": "f577080080471c775e844c519cb94f68d8b4b70dedb33c17f246ede2a99770c0", "shape": [ 34, 33 ] } ], "scheduling_predecessors": [], "semantics": { "embedding_width": 33, "input_name": "embedding_input", "vocab_size": 34 }, "source_nodes": [ "s:8" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:9", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:9" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:10", "inputs": [ "l:11" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3bb9bd83a7c5002d056777dc8133aeefb93900d51f26017a1a809df297d7aa12", "shape": [ 891, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:10" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.40152674999999, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:11", "inputs": [ "l:12", "l:76", "l:211", "l:261", "l:293", "l:327", "l:367", "l:404", "l:444", "l:480", "l:514", "l:547", "l:586", "l:622", "l:642", "l:702", "l:751", "l:799", "l:847", "l:873", "l:890", "l:920", "l:944", "l:968", "l:992", "l:1016", "l:1040" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:11" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.040175249999999, "upper_unbounded": false } }, "width": 891 }, { "annotation": null, "id": "l:12", "inputs": [ "l:13" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:12" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:13", "inputs": [ "l:14", "l:30" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:13" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:14", "inputs": [ "l:15" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:14" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:15", "inputs": [ "l:16", "l:29" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:16" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:16", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:17" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.9956489921875, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:17", "inputs": [ "l:18" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:18" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.35743853290536454, "lower_unbounded": false, "upper": 29.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:18", "inputs": [ "l:19" ], "name": "reciprocal_0_45", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "e1ae2d1ffae9a5e717a562c3a3ed22758427a189561eab446d291d61794a6fca", "shape": [ 45 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "12a7bb89b5494ee4cb4831e9ecb33ff0d5d170c222480f293d1af85996ae08d4", "shape": [ 45, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3a1597c5b00dc1a24568403421c53d3fe42330996ee5f8e7aa3853065db6105b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "d47e14c9cf66b4cb3bca428ef8c914d56ae8c8596823bf1d8a69fcb18f2fefbc", "shape": [ 45, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 45 }, "source_nodes": [ "s:19" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6425614670946355, hi=30.52410519957203))" } ], "claimed_range": { "lower": 0.6425614670946355, "lower_unbounded": false, "upper": 30.52410519957203, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6425614670946355, "lower_unbounded": false, "upper": 30.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:19", "inputs": [ "l:20", "l:21", "l:28" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:20" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:20", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:21" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:21", "inputs": [ "l:22", "l:25", "l:26" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:22" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:22", "inputs": [ "l:23", "l:24" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:23" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:23", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:24" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:24", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:25" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:25", "inputs": [ "l:26", "l:27" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:26" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:26", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "fe8947e9c75de0c7a722d4a0a01bc7acb10be2bfac772df3c42693c0f839bc75", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:27" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:27", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:28" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:28", "inputs": [ "l:26" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:29" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:29", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:31" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:30", "inputs": [ "l:31" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:33" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:31", "inputs": [ "l:32", "l:58", "l:67" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:34" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:32", "inputs": [ "l:33" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:35" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:33", "inputs": [ "l:34", "l:57" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:36" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:34", "inputs": [ "l:35", "l:38", "l:44" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:37" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:35", "inputs": [ "l:36", "l:37" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:38" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:36", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:39" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:37", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:40" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:38", "inputs": [ "l:39", "l:56" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:41" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:39", "inputs": [ "l:40" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:42" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:40", "inputs": [ "l:41", "l:49" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:44" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:41", "inputs": [ "l:42" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9fe7252ae4cff6be4d42fdf46fd404e91cf1503a785d6ee2cfc0779c6e4c7a57", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:45" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:42", "inputs": [ "l:43", "l:45", "l:47" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:47" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:43", "inputs": [ "l:44" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:48" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:44", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1d96963de1e65d345a01d2579d659fb9db80a45a36f4e0e85de3c7801d2f0bbd", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:49" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:45", "inputs": [ "l:46" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:50" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:46", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "556ef51bf9d180c5c5da1df13d26129671a25b8df8ee97141ecdb18681bcf06f", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:51" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:47", "inputs": [ "l:48" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:52" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:48", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9c5442b9d0ffed8d2673a1ace282ad01a661c3a14793f768ed10d421763cc9a3", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:53" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:49", "inputs": [ "l:50" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:54" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:50", "inputs": [ "l:51", "l:54", "l:7" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:55" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:51", "inputs": [ "l:52", "l:53" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:56" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:52", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:57" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:53", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:58" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:54", "inputs": [ "l:7", "l:55" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:59" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:55", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:60" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:56", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:61" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:57", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "f8cab608d0db37b3ad632c5fb81de1037b5e40c21163ca60a6859b38d59493e5", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:62" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:58", "inputs": [ "l:59" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:63" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:59", "inputs": [ "l:60", "l:66" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:64" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:60", "inputs": [ "l:61", "l:64", "l:46" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:65" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:61", "inputs": [ "l:62", "l:63" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:66" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:62", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:67" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:63", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:68" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:64", "inputs": [ "l:39", "l:65" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:69" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:65", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:70" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:66", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "f8cab608d0db37b3ad632c5fb81de1037b5e40c21163ca60a6859b38d59493e5", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:71" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:67", "inputs": [ "l:68" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:72" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:68", "inputs": [ "l:69", "l:75" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:73" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:69", "inputs": [ "l:70", "l:73", "l:48" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:74" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:70", "inputs": [ "l:71", "l:72" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:75" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:71", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:76" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:72", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:77" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:73", "inputs": [ "l:39", "l:74" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:78" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:74", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:79" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:75", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "f8cab608d0db37b3ad632c5fb81de1037b5e40c21163ca60a6859b38d59493e5", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:80" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:76", "inputs": [ "l:77" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:81" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:77", "inputs": [ "l:78", "l:82" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:82" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:78", "inputs": [ "l:79" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:83" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:79", "inputs": [ "l:80", "l:81" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:85" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:80", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:86" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:81", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9bc792642836b330e7b77ccfcb99d0ea4176f98a05cc12439b285610ead907ae", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:87" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:82", "inputs": [ "l:83" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:89" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:83", "inputs": [ "l:84", "l:186", "l:208" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:90" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:84", "inputs": [ "l:85" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:91" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:85", "inputs": [ "l:34", "l:86" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:92" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:86", "inputs": [ "l:87" ], "name": "collapse_Add#1015_fused_fused_add_const_multiply_const_onehot_lookup_select_0_2", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1040ec6986d1e4d299766894e06d24013ef577e8bdffb9a4d6f383b1fa618975", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3602375077f1d703cf16f3ee4d886c01cd94e6f162e87a89cc884dcb8e00fc14", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "305a913fb19882bb387eba7081871b94b962b44421c9cac9371c1a7f61eecf2e", "shape": [ 2, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:93" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906249969, hi=1.0021755039062499))" } ], "claimed_range": { "lower": -0.002175503906249969, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249969, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:87", "inputs": [ "l:88", "l:141" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:99" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:88", "inputs": [ "l:89" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:100" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:89", "inputs": [ "l:90", "l:93", "l:95" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:101" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:90", "inputs": [ "l:91", "l:92" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:102" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:91", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:103" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:92", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:104" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:93", "inputs": [ "l:7", "l:94" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:105" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:94", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:106" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:95", "inputs": [ "l:96", "l:115", "l:138" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:107" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:96", "inputs": [ "l:97" ], "name": "collapse_pl_Attn#667_m675_0_17", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "4a02f2d90e882bbcc9baf4d5ee2c3ee11e4452fded3c1c025541ed7749a28172", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ec23f63c1abc506eff7bd523c8de0d39ff8d1026fc605e895a14c263f015a00b", "shape": [ 17, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50829d67c2320f71da81cbc51e04f11877e2884b2ae9b5e562ad1d5f6f7edc93", "shape": [ 17, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 17 }, "source_nodes": [ "s:108" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009575252532958))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009575252532958, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009575252532958, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:97", "inputs": [ "l:98", "l:101", "l:103" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:115" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:98", "inputs": [ "l:99", "l:100" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:116" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:99", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:117" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:100", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:118" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:101", "inputs": [ "l:39", "l:102" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:119" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:102", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:120" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:103", "inputs": [ "l:104" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:121" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:104", "inputs": [ "l:105" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "86ace85102b9d8a1c0886e9cf2cbb181c89f8ca60093d8ccbe6cbd12a754ad1b", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5870a33a3d94db2db2aaa92ac9edb125e7803a503405c199042a2f0c6d461a6b", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "41d043a7c0f00b8b39f1365727f09ede61e4b865694b6dc1d4acbbd97029d11b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a4d06da2357859d57b942afb3157f9957beb9485b57a8ae112cd6e8f5e6b3764", "shape": [ 32, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "source_nodes": [ "s:122" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6633250767579517, hi=5.503341589908715))" } ], "claimed_range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:105", "inputs": [ "l:106", "l:107", "l:114" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:123" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:106", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:124" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:107", "inputs": [ "l:108", "l:111", "l:112" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:125" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:108", "inputs": [ "l:109", "l:110" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:126" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:109", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:127" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:110", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:128" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:111", "inputs": [ "l:112", "l:113" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:129" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:112", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "315553db8f515ee433f197f38119115bcfc62f6f3224c0ade3022f47076efcf5", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:130" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:113", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:131" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:114", "inputs": [ "l:112" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:132" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:115", "inputs": [ "l:116", "l:135" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:137" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.5118727628515622, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:116", "inputs": [ "l:117", "l:134" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:138" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.0093727628515623, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:117", "inputs": [ "l:118" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "acc2e4c62e9b92c67550621c1d3c42498eab8b87200286819aae131530c4dc41", "shape": [ 3, 4 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e5cd6f63a143d03bf475cd19b883bbdecec4efb62d8303f209b0dde6ae0565cd", "shape": [ 3, 3 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "ba88de7e97f50c001a9d0c4e5c690cd5e4d90c7e5a7b64b96b485a3847228885", "shape": [ 3, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 3 }, "source_nodes": [ "s:139" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.004351007812499974, hi=1.0043510078125))" } ], "claimed_range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:118", "inputs": [ "l:119", "l:129" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:140" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:119", "inputs": [ "l:120" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:141" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:120", "inputs": [ "l:121", "l:122" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:143" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:121", "inputs": [ "l:8" ], "name": "map_to_table", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "2c34ce1df23b838c5abf2a7f6437cca3d3067ed509ff25f11df6b11b582b51eb", "shape": [ 10 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "88eadde2a89aee8855810a153a5cb169a5e92d43ec5b4f1776e6401b5e163fb5", "shape": [ 10, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e7a2f4b7a886aabf181b94c496dabd00bd3f8da32a01f3da03e6d6868564aed6", "shape": [ 10, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 10 }, "source_nodes": [ "s:144" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-21.0, hi=19.0))" } ], "claimed_range": { "lower": -21.0, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:122", "inputs": [ "l:123" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:145" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:123", "inputs": [ "l:124", "l:127", "l:39" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:146" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:124", "inputs": [ "l:125", "l:126" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:147" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:125", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:148" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:126", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:149" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:127", "inputs": [ "l:39", "l:128" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:150" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:128", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:151" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:129", "inputs": [ "l:130" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "fbfe4ca61ec299680879894a16ca776f81f23454312659ac8736249aa1e9722b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:152" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:130", "inputs": [ "l:131" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "source_nodes": [ "s:154" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:131", "inputs": [ "l:132", "l:133" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:155" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:132", "inputs": [ "l:103" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:156" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:133", "inputs": [ "l:132" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:157" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:134", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:158" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:135", "inputs": [ "l:136" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6ad01752d1a3928862bafa2e6bbe6b2df45d614b086e065fdacf50cb7180f6f3", "shape": [ 4, 5 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c91315e209cc3cd1c12fae1264b1597c61d8ddc4fb51cc0941c96fd1be2076b1", "shape": [ 4, 4 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "6e92b4c838f9ab461fc45d72308d5a5360af4497fff417bae914541608979a02", "shape": [ 4, 5 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 4 }, "source_nodes": [ "s:159" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:136", "inputs": [ "l:112", "l:137" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:160" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "l:137", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "024712320e6f0d6455efd690279f94b44c2b7ac0f5351f626965347a9d3d4199", "shape": [ 4 ], "values": [ 0.5, 0.5, 0.5, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:161" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.5, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:138", "inputs": [ "l:139" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:162" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:139", "inputs": [ "l:119", "l:8", "l:140" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:163" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:140", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:164" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:141", "inputs": [ "l:142" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:165" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:142", "inputs": [ "l:143", "l:146", "l:148" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:166" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:143", "inputs": [ "l:144", "l:145" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:167" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:144", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:168" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:145", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:169" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:146", "inputs": [ "l:7", "l:147" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:170" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:147", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:171" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:148", "inputs": [ "l:149", "l:167", "l:183" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:172" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:149", "inputs": [ "l:150" ], "name": "collapse_pl_Attn#615_m623_0_17", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "4a02f2d90e882bbcc9baf4d5ee2c3ee11e4452fded3c1c025541ed7749a28172", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ec23f63c1abc506eff7bd523c8de0d39ff8d1026fc605e895a14c263f015a00b", "shape": [ 17, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50829d67c2320f71da81cbc51e04f11877e2884b2ae9b5e562ad1d5f6f7edc93", "shape": [ 17, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 17 }, "source_nodes": [ "s:173" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009575252532958))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009575252532958, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009575252532958, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:150", "inputs": [ "l:151", "l:154", "l:156" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:180" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:151", "inputs": [ "l:152", "l:153" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:181" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:152", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:182" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:153", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:183" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:154", "inputs": [ "l:7", "l:155" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:184" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:155", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:185" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:156", "inputs": [ "l:157" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:186" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:157", "inputs": [ "l:158" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "86ace85102b9d8a1c0886e9cf2cbb181c89f8ca60093d8ccbe6cbd12a754ad1b", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5870a33a3d94db2db2aaa92ac9edb125e7803a503405c199042a2f0c6d461a6b", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "41d043a7c0f00b8b39f1365727f09ede61e4b865694b6dc1d4acbbd97029d11b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a4d06da2357859d57b942afb3157f9957beb9485b57a8ae112cd6e8f5e6b3764", "shape": [ 32, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "source_nodes": [ "s:187" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6633250767579517, hi=5.503341589908715))" } ], "claimed_range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:158", "inputs": [ "l:159", "l:160", "l:166" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:188" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0021755039062499]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:159", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:189" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:160", "inputs": [ "l:161", "l:164", "l:39" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:190" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:161", "inputs": [ "l:162", "l:163" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:191" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:162", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:192" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:163", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:193" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:164", "inputs": [ "l:39", "l:165" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:194" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:165", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:195" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:166", "inputs": [ "l:39" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:196" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:167", "inputs": [ "l:168", "l:180" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:201" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.5118727628515622, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:168", "inputs": [ "l:169", "l:179" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:202" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.0093727628515623, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:169", "inputs": [ "l:170" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "acc2e4c62e9b92c67550621c1d3c42498eab8b87200286819aae131530c4dc41", "shape": [ 3, 4 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e5cd6f63a143d03bf475cd19b883bbdecec4efb62d8303f209b0dde6ae0565cd", "shape": [ 3, 3 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "ba88de7e97f50c001a9d0c4e5c690cd5e4d90c7e5a7b64b96b485a3847228885", "shape": [ 3, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 3 }, "source_nodes": [ "s:203" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.004351007812499974, hi=1.0043510078125))" } ], "claimed_range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:170", "inputs": [ "l:171", "l:174" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:204" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:171", "inputs": [ "l:172" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f22239942c4ac043eb6887cd996054943f2d24a5e56217c21186a366ca97c23d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9fe7252ae4cff6be4d42fdf46fd404e91cf1503a785d6ee2cfc0779c6e4c7a57", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:205" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:172", "inputs": [ "l:121", "l:123", "l:173" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:207" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:173", "inputs": [ "l:50" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:208" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:174", "inputs": [ "l:175" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "fbfe4ca61ec299680879894a16ca776f81f23454312659ac8736249aa1e9722b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:209" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:175", "inputs": [ "l:176" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "source_nodes": [ "s:211" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:176", "inputs": [ "l:177", "l:178" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:212" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:177", "inputs": [ "l:156" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:213" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:178", "inputs": [ "l:177" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:214" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:179", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:215" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:180", "inputs": [ "l:181" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6ad01752d1a3928862bafa2e6bbe6b2df45d614b086e065fdacf50cb7180f6f3", "shape": [ 4, 5 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c91315e209cc3cd1c12fae1264b1597c61d8ddc4fb51cc0941c96fd1be2076b1", "shape": [ 4, 4 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "6e92b4c838f9ab461fc45d72308d5a5360af4497fff417bae914541608979a02", "shape": [ 4, 5 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 4 }, "source_nodes": [ "s:216" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:181", "inputs": [ "l:39", "l:182" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:217" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "l:182", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "024712320e6f0d6455efd690279f94b44c2b7ac0f5351f626965347a9d3d4199", "shape": [ 4 ], "values": [ 0.5, 0.5, 0.5, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:218" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.5, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:183", "inputs": [ "l:184" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:219" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:184", "inputs": [ "l:171", "l:8", "l:185" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:220" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:185", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:221" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:186", "inputs": [ "l:187" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:224" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:187", "inputs": [ "l:60", "l:188" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:225" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:188", "inputs": [ "l:189" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "bb8fc959ceb3d086d9702adc3f58f39c225857bc7a0f5c41addc2beb5ee1fefd", "shape": [ 3, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:226" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:189", "inputs": [ "l:190" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9c5219c31cbc274e2d589242ce813d94b925fe67b63640f94eef3e6ca029f3bf", "shape": [ 300 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bd021f3ce5360c520035f4f7e6f53fd3221febaa63178551cb2e108177bade04", "shape": [ 300, 69 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3695fc5961dab4c804313662c5662248a3536224fb1dd93f8fb23304c437ff0c", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "5792a0f30ddb29610b18a2899f6a4ddf6c4c4ff947c643fda05bd813e2a31e4c", "shape": [ 300, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 300 }, "source_nodes": [ "s:227" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:190", "inputs": [ "l:191", "l:192", "l:200" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:228" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 69 }, { "annotation": null, "id": "l:191", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "2851d26fefd673cc1134c966bd403171ea775cea78d9c49d1dd489a2931688a8", "shape": [ 33, 3 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:229" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:192", "inputs": [ "l:193", "l:196", "l:198" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:230" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:193", "inputs": [ "l:194", "l:195" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:231" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:194", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:232" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:195", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:233" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:196", "inputs": [ "l:7", "l:197" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:234" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:197", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:235" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:198", "inputs": [ "l:199", "l:115", "l:138" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:236" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:199", "inputs": [ "l:97" ], "name": "collapse_pl_Attn#667_m824_0_9", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "3fdefc9ed00f00dea0599794d1c9a69cee3f0f6849c87508fff2da9d6bbb3ddb", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b05e3b5ab8b2d409e8b3d5f16ea3c8191e3638bbb3168f525a428a2af82fba6b", "shape": [ 9, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "eaa82dd1f86788890259eaf455712f67cccbe5433ad98eac3cad71e322d8d347", "shape": [ 9, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:237" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009574060440063))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:200", "inputs": [ "l:201", "l:204", "l:206" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:246" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:201", "inputs": [ "l:202", "l:203" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:247" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:202", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:248" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:203", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:249" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:204", "inputs": [ "l:7", "l:205" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:250" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:205", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:251" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:206", "inputs": [ "l:207", "l:167", "l:183" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:252" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:207", "inputs": [ "l:150" ], "name": "collapse_pl_Attn#615_m806_0_9", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "3fdefc9ed00f00dea0599794d1c9a69cee3f0f6849c87508fff2da9d6bbb3ddb", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b05e3b5ab8b2d409e8b3d5f16ea3c8191e3638bbb3168f525a428a2af82fba6b", "shape": [ 9, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "eaa82dd1f86788890259eaf455712f67cccbe5433ad98eac3cad71e322d8d347", "shape": [ 9, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:253" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009574060440063))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:208", "inputs": [ "l:209" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:262" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:209", "inputs": [ "l:69", "l:210" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:263" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:210", "inputs": [], "name": "collapse_Linear#701_fused_fused_add_const_multiply_const_onehot_lookup_select", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "3602375077f1d703cf16f3ee4d886c01cd94e6f162e87a89cc884dcb8e00fc14", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:264" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:211", "inputs": [ "l:212" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:277" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:212", "inputs": [ "l:213", "l:217" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:278" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:213", "inputs": [ "l:214" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:279" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:214", "inputs": [ "l:215", "l:216" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:281" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:215", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:282" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:216", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ed53dbef9c852a8229e4b386e20627c3a89599c7583ec50379471fb4415bc5b3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:283" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:217", "inputs": [ "l:218" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:285" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:218", "inputs": [ "l:219", "l:243", "l:248" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:286" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:219", "inputs": [ "l:220" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:287" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:220", "inputs": [ "l:34", "l:221" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:288" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:221", "inputs": [ "l:222" ], "name": "collapse_pl_Add#1222_fused_fused_add_const_multiply_const_onehot_lookup_select_0_10", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "3079f9b91063938a4883906d892f0f04c279a0b60e845dee0d66e762cc2e8edc", "shape": [ 10 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "836fce999334c710891169f4ccdc1eb60a0a1db1212515df25e5b2ddf1b4fbeb", "shape": [ 10, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3602375077f1d703cf16f3ee4d886c01cd94e6f162e87a89cc884dcb8e00fc14", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "7ed88f3221b78ec3bee48f47e585d842287e93540ba95ab3a6e482014f73c6db", "shape": [ 10, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 10 }, "source_nodes": [ "s:289" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.000507596284059433, hi=1.0005037921208524))" } ], "claimed_range": { "lower": -0.000507596284059433, "lower_unbounded": false, "upper": 1.0005037921208524, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.000507596284059433, "lower_unbounded": false, "upper": 1.0005037921208524, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:222", "inputs": [ "l:223", "l:242" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:294" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 24.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:223", "inputs": [ "l:224", "l:233" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:295" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:224", "inputs": [ "l:225" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:296" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:225", "inputs": [ "l:226", "l:229", "l:231" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:297" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:226", "inputs": [ "l:227", "l:228" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:298" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:227", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:299" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:228", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:300" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:229", "inputs": [ "l:7", "l:230" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:301" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:230", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:302" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:231", "inputs": [ "l:232", "l:115", "l:138" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:303" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:232", "inputs": [ "l:97" ], "name": "collapse_pl_Attn#667_m746_0_13", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "01afa9575aaff88a9f3a51a4578114a24a66192bf8355f33ac56fee5616b18ca", "shape": [ 13 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "fb694bb84d282aed16c503a31849c67f82e5982a0fcf9ac863c4f3998a513076", "shape": [ 13, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "1f400d61c0e178ba2a3d7b83d01a8a931c0d44dc581ca40c6549b6acb82fb385", "shape": [ 13, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 13 }, "source_nodes": [ "s:304" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009574060440063))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:233", "inputs": [ "l:234" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:313" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:234", "inputs": [ "l:235", "l:238", "l:240" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:314" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:235", "inputs": [ "l:236", "l:237" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:315" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:236", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:316" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:237", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:317" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:238", "inputs": [ "l:7", "l:239" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:318" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:239", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:319" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:240", "inputs": [ "l:241", "l:167", "l:183" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:320" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:241", "inputs": [ "l:150" ], "name": "collapse_pl_Attn#615_m730_0_13", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "01afa9575aaff88a9f3a51a4578114a24a66192bf8355f33ac56fee5616b18ca", "shape": [ 13 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "fb694bb84d282aed16c503a31849c67f82e5982a0fcf9ac863c4f3998a513076", "shape": [ 13, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "1f400d61c0e178ba2a3d7b83d01a8a931c0d44dc581ca40c6549b6acb82fb385", "shape": [ 13, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 13 }, "source_nodes": [ "s:321" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009574060440063))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:242", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:330" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:243", "inputs": [ "l:244" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:332" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:244", "inputs": [ "l:60", "l:245" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:333" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:245", "inputs": [ "l:246" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "bb8fc959ceb3d086d9702adc3f58f39c225857bc7a0f5c41addc2beb5ee1fefd", "shape": [ 3, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:334" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:246", "inputs": [ "l:247" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9c5219c31cbc274e2d589242ce813d94b925fe67b63640f94eef3e6ca029f3bf", "shape": [ 300 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bd021f3ce5360c520035f4f7e6f53fd3221febaa63178551cb2e108177bade04", "shape": [ 300, 69 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3695fc5961dab4c804313662c5662248a3536224fb1dd93f8fb23304c437ff0c", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "5792a0f30ddb29610b18a2899f6a4ddf6c4c4ff947c643fda05bd813e2a31e4c", "shape": [ 300, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 300 }, "source_nodes": [ "s:335" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:247", "inputs": [ "l:191", "l:225", "l:234" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:336" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 69 }, { "annotation": null, "id": "l:248", "inputs": [ "l:249" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:337" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:249", "inputs": [ "l:69", "l:250" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:338" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:250", "inputs": [ "l:251" ], "name": "collapse_pl_Add#1210_fused_fused_add_const_multiply_const_onehot_lookup_select_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f5b0e2448da4389debb5a8bb83ed88279b196b72efb086ac626c36d490b978fe", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "4531ea6022a4b7bb47709567b2fe5929875ff67938ca616a1be1d276db19809c", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3602375077f1d703cf16f3ee4d886c01cd94e6f162e87a89cc884dcb8e00fc14", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e647aedfda2362a943e2b71acd083ea4ba24836286b6615cf83369ae5831eed5", "shape": [ 14, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:339" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005151736478389455, hi=1.0005151736478388))" } ], "claimed_range": { "lower": -0.0005151736478389455, "lower_unbounded": false, "upper": 1.0005151736478388, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005151736478389455, "lower_unbounded": false, "upper": 1.0005151736478388, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:251", "inputs": [ "l:252", "l:260" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:344" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:252", "inputs": [ "l:253" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "a906259dd1919a62fdac039d61961e0f375aef08161ea9d5be01e40a036fe21e", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:345" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:253", "inputs": [ "l:254", "l:256", "l:258" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:254", "inputs": [ "l:255" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:348" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:255", "inputs": [ "l:225", "l:142" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:349" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "l:256", "inputs": [ "l:257" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:351" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:257", "inputs": [ "l:89", "l:234" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:352" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "l:258", "inputs": [ "l:259" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:273" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:259", "inputs": [ "l:89", "l:142" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:274" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "l:260", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:354" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:261", "inputs": [ "l:262" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:356" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:262", "inputs": [ "l:263", "l:267" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:357" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:263", "inputs": [ "l:264" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:358" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:264", "inputs": [ "l:265", "l:266" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:360" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:265", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:361" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:266", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9dfa5cc0a4093ad092e8327b9861cf4f1595c50fa4b3f12efc578b78fe5afdd2", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:362" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:267", "inputs": [ "l:268" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:364" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:268", "inputs": [ "l:269", "l:276", "l:281" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:365" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:269", "inputs": [ "l:270" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:366" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:270", "inputs": [ "l:34", "l:271" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:367" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:271", "inputs": [ "l:272" ], "name": "collapse_pl_Add#1195_fused_fused_add_const_multiply_const_onehot_lookup_select_0_10", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "3079f9b91063938a4883906d892f0f04c279a0b60e845dee0d66e762cc2e8edc", "shape": [ 10 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "836fce999334c710891169f4ccdc1eb60a0a1db1212515df25e5b2ddf1b4fbeb", "shape": [ 10, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3602375077f1d703cf16f3ee4d886c01cd94e6f162e87a89cc884dcb8e00fc14", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "7ed88f3221b78ec3bee48f47e585d842287e93540ba95ab3a6e482014f73c6db", "shape": [ 10, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 10 }, "source_nodes": [ "s:368" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.000507596284059433, hi=1.0005037921208524))" } ], "claimed_range": { "lower": -0.000507596284059433, "lower_unbounded": false, "upper": 1.0005037921208524, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.000507596284059433, "lower_unbounded": false, "upper": 1.0005037921208524, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:272", "inputs": [ "l:273", "l:242" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:373" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 24.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:273", "inputs": [ "l:274", "l:275" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:374" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:274", "inputs": [ "l:192" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:375" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:275", "inputs": [ "l:200" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:376" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:276", "inputs": [ "l:277" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:378" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:277", "inputs": [ "l:60", "l:278" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:379" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:278", "inputs": [ "l:279" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "bb8fc959ceb3d086d9702adc3f58f39c225857bc7a0f5c41addc2beb5ee1fefd", "shape": [ 3, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:380" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:279", "inputs": [ "l:280" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9c5219c31cbc274e2d589242ce813d94b925fe67b63640f94eef3e6ca029f3bf", "shape": [ 300 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bd021f3ce5360c520035f4f7e6f53fd3221febaa63178551cb2e108177bade04", "shape": [ 300, 69 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3695fc5961dab4c804313662c5662248a3536224fb1dd93f8fb23304c437ff0c", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "5792a0f30ddb29610b18a2899f6a4ddf6c4c4ff947c643fda05bd813e2a31e4c", "shape": [ 300, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 300 }, "source_nodes": [ "s:381" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:280", "inputs": [ "l:191", "l:89", "l:142" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:382" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 69 }, { "annotation": null, "id": "l:281", "inputs": [ "l:282" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:383" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:282", "inputs": [ "l:69", "l:283" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:384" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:283", "inputs": [ "l:284" ], "name": "collapse_pl_Add#1182_fused_fused_add_const_multiply_const_onehot_lookup_select_0_22", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "343abce6f173be2450a7ce7e705e371ef8d8c7d4d109911a0f41ffa35258b173", "shape": [ 22 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "24b4a5993c07fd1e4e2bbe77152bb92758a56f0ce4deee9cecf0f3b04b1f4949", "shape": [ 22, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3602375077f1d703cf16f3ee4d886c01cd94e6f162e87a89cc884dcb8e00fc14", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "dbff3eb89136898213322583ecb0428dc70516754ab928746875b4b1987ad893", "shape": [ 22, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 22 }, "source_nodes": [ "s:385" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005152286915907535, hi=1.0005151684966471))" } ], "claimed_range": { "lower": -0.0005152286915907535, "lower_unbounded": false, "upper": 1.0005151684966471, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005152286915907535, "lower_unbounded": false, "upper": 1.0005151684966471, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:284", "inputs": [ "l:285", "l:260" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:390" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 51.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:285", "inputs": [ "l:286" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d67537e14beb321ce93c69b4a330880e5d3ea3712a2e57015626869e0b444413", "shape": [ 10, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:391" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:286", "inputs": [ "l:287", "l:289", "l:254", "l:291", "l:256" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 10 }, { "annotation": null, "id": "l:287", "inputs": [ "l:288" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:394" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:288", "inputs": [ "l:192", "l:142" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:395" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "l:289", "inputs": [ "l:290" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:397" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:290", "inputs": [ "l:225", "l:234" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:398" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "l:291", "inputs": [ "l:292" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:401" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:292", "inputs": [ "l:89", "l:200" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:402" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "l:293", "inputs": [ "l:294" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:405" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:294", "inputs": [ "l:295", "l:299" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:406" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:295", "inputs": [ "l:296" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:407" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:296", "inputs": [ "l:297", "l:298" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:409" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:297", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:410" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:298", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25eab693bdf7c6ad20a03812dd32b8a7ba27dab587478a5dc1c8b657174ae97d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:411" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:299", "inputs": [ "l:300" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:413" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:300", "inputs": [ "l:301", "l:304", "l:317" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:414" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:301", "inputs": [ "l:302" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:415" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:302", "inputs": [ "l:34", "l:303" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:416" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:303", "inputs": [], "name": "collapse_pl_onehot_lookup_select_fused_fused_add_const_multiply_const_onehot_lookup_select", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "3602375077f1d703cf16f3ee4d886c01cd94e6f162e87a89cc884dcb8e00fc14", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:417" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:304", "inputs": [ "l:305" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:425" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:305", "inputs": [ "l:60", "l:306" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:426" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:306", "inputs": [ "l:307" ], "name": "collapse_pl_Add#1157_fused_fused_add_const_multiply_const_onehot_lookup_select_0_4", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5ab50eef4b2f7f817202a823bd39216fe6a98ada56b2f08dab1ee8f88cedf890", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "cb2d0756b6fc5311ca9f1fe6ca379c7a6618cf1e1a6ec08d25ed4dde93afb2af", "shape": [ 4, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "b9a731d73ddd5c1389f8d60c7323707bc72e2c505fa26496b0d850e5fc81a033", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "0702c768e237b460e8b57c666685fb0d71fb296de8b4794baffffb899a1ff875", "shape": [ 4, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 4 }, "source_nodes": [ "s:427" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005, hi=1.0005))" } ], "claimed_range": { "lower": -0.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:307", "inputs": [ "l:308", "l:314" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:434" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:308", "inputs": [ "l:309" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "source_nodes": [ "s:435" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:309", "inputs": [ "l:310", "l:312", "l:313" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:436" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:310", "inputs": [ "l:311" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:437" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:311", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:438" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:312", "inputs": [ "l:89" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:439" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:313", "inputs": [ "l:142" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:440" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:314", "inputs": [ "l:315" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:441" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:315", "inputs": [ "l:316" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "source_nodes": [ "s:442" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:316", "inputs": [ "l:310", "l:313", "l:312" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:443" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:317", "inputs": [ "l:318" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:447" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:318", "inputs": [ "l:69", "l:319" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:448" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:319", "inputs": [ "l:320" ], "name": "collapse_pl_Add#1141_fused_fused_add_const_multiply_const_onehot_lookup_select_0_22", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "343abce6f173be2450a7ce7e705e371ef8d8c7d4d109911a0f41ffa35258b173", "shape": [ 22 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "24b4a5993c07fd1e4e2bbe77152bb92758a56f0ce4deee9cecf0f3b04b1f4949", "shape": [ 22, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3602375077f1d703cf16f3ee4d886c01cd94e6f162e87a89cc884dcb8e00fc14", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "dbff3eb89136898213322583ecb0428dc70516754ab928746875b4b1987ad893", "shape": [ 22, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 22 }, "source_nodes": [ "s:449" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005152286915907535, hi=1.0005151684966471))" } ], "claimed_range": { "lower": -0.0005152286915907535, "lower_unbounded": false, "upper": 1.0005151684966471, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005152286915907535, "lower_unbounded": false, "upper": 1.0005151684966471, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:320", "inputs": [ "l:321", "l:260" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:454" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 51.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:321", "inputs": [ "l:322" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "5e85f9ba49a0af5e1df5a336fb36c8cb22670d559598896896029f6c7f8b2e71", "shape": [ 10, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:455" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:322", "inputs": [ "l:323", "l:287", "l:325", "l:289", "l:291" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 10 }, { "annotation": null, "id": "l:323", "inputs": [ "l:324" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:458" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:324", "inputs": [ "l:192", "l:234" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:459" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "l:325", "inputs": [ "l:326" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:462" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:326", "inputs": [ "l:225", "l:200" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:463" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "l:327", "inputs": [ "l:328" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:467" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:328", "inputs": [ "l:329", "l:333" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:468" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:329", "inputs": [ "l:330" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:469" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:330", "inputs": [ "l:331", "l:332" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:471" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:331", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "db8d1a556ec9527f63930661298f4b8607cfcecb3c364235751acfef1ec8ed1a", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:472" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:332", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "64f403ba374325ca540cb27c0375a0ea1dc20ecaeb654fcbe8bea2b756826dab", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:473" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:333", "inputs": [ "l:334" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:475" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:334", "inputs": [ "l:335", "l:341", "l:359" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:476" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:335", "inputs": [ "l:336" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:477" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:336", "inputs": [ "l:34", "l:337" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:478" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:337", "inputs": [ "l:338" ], "name": "collapse_pl_onehot_lookup_select_fused_fused_add_const_multiply_const_onehot_lookup_select_0_26", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ce9c6c63ab9421b9208a2043fd9fcce69be76f1bf392b4d40e7430ac5d66fdfc", "shape": [ 26 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "c53e742ec5155f4c8605612cf890940743ae14e7fce502464da7d30f02b30bd2", "shape": [ 26, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "cb7bb1c4a70e818492fb5cebc635b1704da80e10fda9104cb4c066ddeaf5795c", "shape": [ 26, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 26 }, "source_nodes": [ "s:479" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005018659575333207, hi=1.0005036656891495))" } ], "claimed_range": { "lower": -0.0005018659575333207, "lower_unbounded": false, "upper": 1.0005036656891495, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005018659575333207, "lower_unbounded": false, "upper": 1.0005036656891495, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:338", "inputs": [ "l:339" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:485" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:339", "inputs": [ "l:340", "l:340", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:486" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:340", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:487" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:341", "inputs": [ "l:342" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:489" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:342", "inputs": [ "l:60", "l:343" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:490" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:343", "inputs": [ "l:344" ], "name": "collapse_pl_Add#1114_fused_fused_add_const_multiply_const_onehot_lookup_select_0_9", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0f12b609444d9d659ea55d7f93501e3a5f15753052bf7e35502e53186380f603", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ed8a0254b2ee4f83693917f79459fdc3e872c367d5174798ae5c53fe934f0580", "shape": [ 9, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "8da8a5ffcfb3dc7d79092b32fb5d267824d0957065534e11ae5d47629c54d205", "shape": [ 9, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:491" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005, hi=1.0005047589647418))" } ], "claimed_range": { "lower": -0.0005, "lower_unbounded": false, "upper": 1.0005047589647418, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005, "lower_unbounded": false, "upper": 1.0005047589647418, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:344", "inputs": [ "l:345", "l:357" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:497" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -15.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:345", "inputs": [ "l:346", "l:354" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:498" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:346", "inputs": [ "l:347" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "source_nodes": [ "s:499" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:347", "inputs": [ "l:348", "l:352", "l:353" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:500" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:348", "inputs": [ "l:349" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:501" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:349", "inputs": [ "l:350" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:502" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:350", "inputs": [ "l:351", "l:351", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:503" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:351", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:504" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:352", "inputs": [ "l:225" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:505" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:353", "inputs": [ "l:234" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:506" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:354", "inputs": [ "l:355" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:507" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:355", "inputs": [ "l:356" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "source_nodes": [ "s:508" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:356", "inputs": [ "l:348", "l:353", "l:352" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:509" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:357", "inputs": [ "l:358" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:510" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -6.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:358", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:511" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:359", "inputs": [ "l:360" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:513" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:360", "inputs": [ "l:69", "l:361" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:514" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:361", "inputs": [ "l:362" ], "name": "collapse_pl_Add#1096_fused_fused_add_const_multiply_const_onehot_lookup_select_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f5b0e2448da4389debb5a8bb83ed88279b196b72efb086ac626c36d490b978fe", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "4531ea6022a4b7bb47709567b2fe5929875ff67938ca616a1be1d276db19809c", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3602375077f1d703cf16f3ee4d886c01cd94e6f162e87a89cc884dcb8e00fc14", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e647aedfda2362a943e2b71acd083ea4ba24836286b6615cf83369ae5831eed5", "shape": [ 14, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:515" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005151736478389455, hi=1.0005151736478388))" } ], "claimed_range": { "lower": -0.0005151736478389455, "lower_unbounded": false, "upper": 1.0005151736478388, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005151736478389455, "lower_unbounded": false, "upper": 1.0005151736478388, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:362", "inputs": [ "l:363", "l:260" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:520" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:363", "inputs": [ "l:364" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "a17e23a9a0ac633000f4ef1d56a17483b909274a6270009a5915bba0a164881d", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:521" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:364", "inputs": [ "l:365", "l:323", "l:325" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:365", "inputs": [ "l:366" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:524" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:366", "inputs": [ "l:192", "l:200" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:525" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "l:367", "inputs": [ "l:368" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:529" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:368", "inputs": [ "l:369", "l:373" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:530" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:369", "inputs": [ "l:370" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:531" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:370", "inputs": [ "l:371", "l:372" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:533" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:371", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b8bb7a27e55e02d67b43582292d3134e5715248dc3f4a54dac7e1c71519551b5", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:534" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:372", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df1b487fa89261e6181b4f1197a2248f295f75619fa52d6c77c2a438a9f5862d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:535" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:373", "inputs": [ "l:374" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:537" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:374", "inputs": [ "l:375", "l:382", "l:399" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:538" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:375", "inputs": [ "l:376" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:539" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:376", "inputs": [ "l:34", "l:377" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:540" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:377", "inputs": [ "l:378" ], "name": "collapse_pl_Add#1081_fused_fused_add_const_multiply_const_onehot_lookup_select_0_82", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "3cc06c54716dc70bc2eb5c21c7e655b8fc091c01e26993511b703b3188048e95", "shape": [ 82 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d3ec795fdc4c64a8f73a35f488ba7b567ddc59fe9e276fc197dab83d783f15a2", "shape": [ 82, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9960d4e2f09297f610709329199ce785ab9c92b0a8c5147a11cc4c238ad41af1", "shape": [ 82, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 82 }, "source_nodes": [ "s:541" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.000507596284059433, hi=1.0005074638340876))" } ], "claimed_range": { "lower": -0.000507596284059433, "lower_unbounded": false, "upper": 1.0005074638340876, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.000507596284059433, "lower_unbounded": false, "upper": 1.0005074638340876, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:378", "inputs": [ "l:273", "l:379" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:546" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 24.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:379", "inputs": [ "l:380" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:547" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:380", "inputs": [ "l:381", "l:381", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:548" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:381", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:549" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:382", "inputs": [ "l:383" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:551" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:383", "inputs": [ "l:60", "l:384" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:552" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:384", "inputs": [ "l:385" ], "name": "collapse_pl_Add#1067_fused_fused_add_const_multiply_const_onehot_lookup_select_0_9", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0f12b609444d9d659ea55d7f93501e3a5f15753052bf7e35502e53186380f603", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ed8a0254b2ee4f83693917f79459fdc3e872c367d5174798ae5c53fe934f0580", "shape": [ 9, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "8da8a5ffcfb3dc7d79092b32fb5d267824d0957065534e11ae5d47629c54d205", "shape": [ 9, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:553" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005, hi=1.0005047589647418))" } ], "claimed_range": { "lower": -0.0005, "lower_unbounded": false, "upper": 1.0005047589647418, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005, "lower_unbounded": false, "upper": 1.0005047589647418, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:385", "inputs": [ "l:386", "l:398" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:559" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -15.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:386", "inputs": [ "l:387", "l:395" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:560" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:387", "inputs": [ "l:388" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "source_nodes": [ "s:561" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:388", "inputs": [ "l:389", "l:393", "l:394" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:562" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:389", "inputs": [ "l:390" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:563" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:390", "inputs": [ "l:391" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:564" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:391", "inputs": [ "l:392", "l:392", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:565" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:392", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:566" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:393", "inputs": [ "l:192" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:567" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:394", "inputs": [ "l:200" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:568" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:395", "inputs": [ "l:396" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:569" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:396", "inputs": [ "l:397" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "source_nodes": [ "s:570" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:397", "inputs": [ "l:389", "l:394", "l:393" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:571" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:398", "inputs": [ "l:358" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:572" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -6.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:399", "inputs": [ "l:400" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:574" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:400", "inputs": [ "l:69", "l:401" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:575" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:401", "inputs": [ "l:402" ], "name": "collapse_pl_Add#1048_fused_fused_add_const_multiply_const_onehot_lookup_select_0_6", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "66ec169319d69f345048914335bbbd71001516b5b3f3bcdae1301dc2095a19ec", "shape": [ 6 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "296ead42da6cc235c6051459f71e00746dcfee3f75956f0fa17d0e815d41c8a0", "shape": [ 6, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3602375077f1d703cf16f3ee4d886c01cd94e6f162e87a89cc884dcb8e00fc14", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "b06b00567c70bbc2bc717700cf2df749e73dd3dc7ce61383446c9038df698e8d", "shape": [ 6, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 6 }, "source_nodes": [ "s:576" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005038011514145507, hi=1.0005038011514145))" } ], "claimed_range": { "lower": -0.0005038011514145507, "lower_unbounded": false, "upper": 1.0005038011514145, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005038011514145507, "lower_unbounded": false, "upper": 1.0005038011514145, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:402", "inputs": [ "l:403", "l:260" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:581" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 15.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:403", "inputs": [ "l:365" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:582" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:404", "inputs": [ "l:405" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:586" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:405", "inputs": [ "l:406", "l:410" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:587" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:406", "inputs": [ "l:407" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:588" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:407", "inputs": [ "l:408", "l:409" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:590" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:408", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "fcc789a616d690b13c0ce2bb32111dec233ff36515f19e597b74cfe3b5615bcd", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:591" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:409", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ee1c1e99aec431b3878503f437c2d6ed686bab105f28857153577a62843be75e", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:592" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:410", "inputs": [ "l:411" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:594" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:411", "inputs": [ "l:412", "l:419", "l:437" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:595" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:412", "inputs": [ "l:413" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:596" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:413", "inputs": [ "l:34", "l:414" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:597" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:414", "inputs": [ "l:415" ], "name": "collapse_pl_Add#1032_fused_fused_add_const_multiply_const_onehot_lookup_select_0_82", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "3cc06c54716dc70bc2eb5c21c7e655b8fc091c01e26993511b703b3188048e95", "shape": [ 82 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d3ec795fdc4c64a8f73a35f488ba7b567ddc59fe9e276fc197dab83d783f15a2", "shape": [ 82, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9960d4e2f09297f610709329199ce785ab9c92b0a8c5147a11cc4c238ad41af1", "shape": [ 82, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 82 }, "source_nodes": [ "s:598" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.000507596284059433, hi=1.0005074638340876))" } ], "claimed_range": { "lower": -0.000507596284059433, "lower_unbounded": false, "upper": 1.0005074638340876, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.000507596284059433, "lower_unbounded": false, "upper": 1.0005074638340876, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:415", "inputs": [ "l:223", "l:416" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:603" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 24.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:416", "inputs": [ "l:417" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:604" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:417", "inputs": [ "l:418", "l:418", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:605" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:418", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:606" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:419", "inputs": [ "l:420" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:608" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:420", "inputs": [ "l:60", "l:421" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:609" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:421", "inputs": [ "l:422" ], "name": "collapse_pl_Add#1018_fused_fused_add_const_multiply_const_onehot_lookup_select_0_81", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ab318b3284838ec49f48983576a845adef8f2907c8b04b38e9dcfe0b699378c7", "shape": [ 81 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "402c31b48770892ab608c50ea623b543e81bedc1e1891583fb0515124170a440", "shape": [ 81, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "dafa460c7df38b10724e8f39d898a6181266668e7c8e4ace0bf44c5fc58de64f", "shape": [ 81, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 81 }, "source_nodes": [ "s:610" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005046648963048526, hi=1.0005046028111384))" } ], "claimed_range": { "lower": -0.0005046648963048526, "lower_unbounded": false, "upper": 1.0005046028111384, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005046648963048526, "lower_unbounded": false, "upper": 1.0005046028111384, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:422", "inputs": [ "l:423", "l:433" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:616" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -15.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:423", "inputs": [ "l:424", "l:430" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:617" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:424", "inputs": [ "l:425" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "source_nodes": [ "s:618" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:425", "inputs": [ "l:426", "l:393", "l:394" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:619" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:426", "inputs": [ "l:427" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:620" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:427", "inputs": [ "l:428" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:621" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:428", "inputs": [ "l:429", "l:429", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:622" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:429", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:623" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:430", "inputs": [ "l:431" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:624" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:431", "inputs": [ "l:432" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "source_nodes": [ "s:625" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:432", "inputs": [ "l:426", "l:394", "l:393" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:626" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:433", "inputs": [ "l:434" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:627" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -6.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:434", "inputs": [ "l:435" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:628" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:435", "inputs": [ "l:436", "l:436", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:629" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:436", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:630" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:437", "inputs": [ "l:438" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:632" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:438", "inputs": [ "l:69", "l:439" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:633" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:439", "inputs": [ "l:440" ], "name": "collapse_pl_Add#995_fused_fused_add_const_multiply_const_onehot_lookup_select_0_62", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "88508c266e465aa4db45d5aee214f0a82f7d17ab8babd9a73f2ee21d8016d654", "shape": [ 62 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "948ab21a5eb08ecd1a0cee73a52b3545d323f52603ac2a9d8efb085fd3d6358b", "shape": [ 62, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9f8688b5881b3ec866e1b07f4392adc8c712b11c049cb52452e5992c446ccd18", "shape": [ 62, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 62 }, "source_nodes": [ "s:634" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005074638340877642, hi=1.0005074638340876))" } ], "claimed_range": { "lower": -0.0005074638340877642, "lower_unbounded": false, "upper": 1.0005074638340876, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005074638340877642, "lower_unbounded": false, "upper": 1.0005074638340876, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:440", "inputs": [ "l:403", "l:441" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:639" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 15.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:441", "inputs": [ "l:442" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:640" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:442", "inputs": [ "l:443", "l:443", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:641" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:443", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:642" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:444", "inputs": [ "l:445" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:644" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:445", "inputs": [ "l:446", "l:450" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:645" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:446", "inputs": [ "l:447" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:646" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:447", "inputs": [ "l:448", "l:449" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:648" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:448", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "bd2babfb9c1d66c869f90c4f5d274aa44df07a3fabd7bafefbc4f8da66c7ffad", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:649" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:449", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "e1eb8965d53f3eec81b3ef488acc3641be29be59d2f6e900aba04da322a11aa7", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:650" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:450", "inputs": [ "l:451" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:652" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:451", "inputs": [ "l:452", "l:455", "l:473" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:653" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:452", "inputs": [ "l:453" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:654" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:453", "inputs": [ "l:34", "l:454" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:655" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:454", "inputs": [ "l:87" ], "name": "collapse_Add#1015_fused_fused_add_const_multiply_const_onehot_lookup_select_0_36", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "33d6083c98dc75a96670fd1422bf103411014e4867ab32c72a9942251dfebe21", "shape": [ 36 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a4624f63c3af1653c3d97399a5161fc2aa9b2ba4196c73790409b866c110c8b1", "shape": [ 36, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "5dffd7fbef23daa438116335ddca30c7de637678c9655bccc99639157aa08f13", "shape": [ 36, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 36 }, "source_nodes": [ "s:656" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:455", "inputs": [ "l:456" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:664" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:456", "inputs": [ "l:60", "l:457" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:665" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:457", "inputs": [ "l:458" ], "name": "collapse_pl_Add#969_fused_fused_add_const_multiply_const_onehot_lookup_select_0_81", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ab318b3284838ec49f48983576a845adef8f2907c8b04b38e9dcfe0b699378c7", "shape": [ 81 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "402c31b48770892ab608c50ea623b543e81bedc1e1891583fb0515124170a440", "shape": [ 81, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "dafa460c7df38b10724e8f39d898a6181266668e7c8e4ace0bf44c5fc58de64f", "shape": [ 81, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 81 }, "source_nodes": [ "s:666" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005046648963048526, hi=1.0005046028111384))" } ], "claimed_range": { "lower": -0.0005046648963048526, "lower_unbounded": false, "upper": 1.0005046028111384, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005046648963048526, "lower_unbounded": false, "upper": 1.0005046028111384, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:458", "inputs": [ "l:459", "l:469" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:672" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -15.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:459", "inputs": [ "l:460", "l:466" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:673" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:460", "inputs": [ "l:461" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "source_nodes": [ "s:674" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:461", "inputs": [ "l:462", "l:352", "l:353" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:675" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:462", "inputs": [ "l:463" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:676" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:463", "inputs": [ "l:464" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:677" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:464", "inputs": [ "l:465", "l:465", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "4a219c06af2467623290b375e580c952ecdae33edf14aa594ac393d27e76b67e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:678" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:465", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:679" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:466", "inputs": [ "l:467" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:680" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:467", "inputs": [ "l:468" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "source_nodes": [ "s:681" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:468", "inputs": [ "l:462", "l:353", "l:352" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:682" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:469", "inputs": [ "l:470" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:683" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -6.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:470", "inputs": [ "l:471" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:684" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:471", "inputs": [ "l:472", "l:472", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:685" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:472", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:686" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:473", "inputs": [ "l:474" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:688" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:474", "inputs": [ "l:69", "l:475" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:689" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:475", "inputs": [ "l:476" ], "name": "collapse_pl_Add#946_fused_fused_add_const_multiply_const_onehot_lookup_select_0_134", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "4e705c585d757a36606f6a63afbf953330905910ea83d6b28ffce0c96629c8b7", "shape": [ 134 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "0cba320c895fe5f239fd4b74d5a47ac0c4fee5f799cfc8f13aa74af1c53d4596", "shape": [ 134, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f011a372d523de15d7790b8257097a89fe241680948311d06d2d5357992cbc3d", "shape": [ 134, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 134 }, "source_nodes": [ "s:690" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005149276857477295, hi=1.0005149276843444))" } ], "claimed_range": { "lower": -0.0005149276857477295, "lower_unbounded": false, "upper": 1.0005149276843444, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005149276857477295, "lower_unbounded": false, "upper": 1.0005149276843444, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:476", "inputs": [ "l:363", "l:477" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:695" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:477", "inputs": [ "l:478" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:696" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:478", "inputs": [ "l:479", "l:479", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:697" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:479", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:698" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:480", "inputs": [ "l:481" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:700" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:481", "inputs": [ "l:482", "l:486" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:701" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:482", "inputs": [ "l:483" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:702" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:483", "inputs": [ "l:484", "l:485" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:704" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:484", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b523f26d8d2fa76eb7653ecf593398e72ab630efde060b02aa9e7744b3faa6e0", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:705" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:485", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "46e48c3d02450eeca0c725e363868a61942d805b81019328f5624cc7be3e802b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:706" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:486", "inputs": [ "l:487" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:708" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:487", "inputs": [ "l:488", "l:494", "l:507" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:709" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:488", "inputs": [ "l:489" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:710" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:489", "inputs": [ "l:34", "l:490" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:711" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:490", "inputs": [ "l:491" ], "name": "collapse_pl_equals_vector_fused_fused_add_const_multiply_const_onehot_lookup_select_0_5", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "e491d258ef7d16045840f85b621ada893b0f35f02ae622c89cc012f17e89b5fa", "shape": [ 5 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b27cd9af0efbe781be2ba8a2bcfca1dfb496151846378a5d9ebca516d75d947d", "shape": [ 5, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "09511141e3c1dbec582737bc8323c4554fd8c7cc2946636288ea30de8cf24e53", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "d53a49752f4fdd093968330713c0411479789f5462a0ddfef21ff831a000b9cd", "shape": [ 5, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 5 }, "source_nodes": [ "s:712" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005019037815396516, hi=1.0005019037815397))" } ], "claimed_range": { "lower": -0.0005019037815396516, "lower_unbounded": false, "upper": 1.0005019037815397, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005019037815396516, "lower_unbounded": false, "upper": 1.0005019037815397, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:491", "inputs": [ "l:492" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:725" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:492", "inputs": [ "l:493", "l:493", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:726" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:493", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:727" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:494", "inputs": [ "l:495" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:729" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:495", "inputs": [ "l:60", "l:496" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:730" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:496", "inputs": [ "l:497" ], "name": "collapse_pl_Add#907_fused_fused_add_const_multiply_const_onehot_lookup_select_0_73", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6994cf6394c512d0e0ace17b73ec06b2f046ac135f130a329fdabd80e3e0d6ed", "shape": [ 73 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3f43162ed713ed42ec851e8f90cf45af41dd203e91a78ba26ffc97ad8dfe1463", "shape": [ 73, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "8d4850fba7d5521cde31e3b82cb005c00918d43f42ccb293dce7690ef0270666", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9ede12aab6c34f9d6c52797751c555636539176fcaaaadb5ca9cbd84450b99c4", "shape": [ 73, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 73 }, "source_nodes": [ "s:731" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005046648963048526, hi=1.0005046028111384))" } ], "claimed_range": { "lower": -0.0005046648963048526, "lower_unbounded": false, "upper": 1.0005046028111384, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005046648963048526, "lower_unbounded": false, "upper": 1.0005046028111384, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:497", "inputs": [ "l:498", "l:504" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:738" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:498", "inputs": [ "l:499" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "source_nodes": [ "s:739" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:499", "inputs": [ "l:500", "l:312", "l:313" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:740" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:500", "inputs": [ "l:501" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:741" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:501", "inputs": [ "l:502" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:742" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:502", "inputs": [ "l:503", "l:503", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:743" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:503", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:744" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:504", "inputs": [ "l:505" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:745" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:505", "inputs": [ "l:506" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "source_nodes": [ "s:746" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:506", "inputs": [ "l:500", "l:313", "l:312" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:747" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:507", "inputs": [ "l:508" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:751" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:508", "inputs": [ "l:69", "l:509" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:752" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:509", "inputs": [ "l:510" ], "name": "collapse_pl_Add#887_fused_fused_add_const_multiply_const_onehot_lookup_select_0_206", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "27ecd8d43ef3729b3c70d8fc52d57f7c663ba44cec39e041415559617817abac", "shape": [ 206 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bd1a54bf1f24b30e5c23b5d0d6fc207a15657a73c529b3e3898c768de8f32ad4", "shape": [ 206, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "583141be6edfe40ceefc0f51e4a1071c42295c53820b499aa40b1f4275435486", "shape": [ 206, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 206 }, "source_nodes": [ "s:753" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005149277173835658, hi=1.0005292111554163))" } ], "claimed_range": { "lower": -0.0005149277173835658, "lower_unbounded": false, "upper": 1.0005292111554163, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005149277173835658, "lower_unbounded": false, "upper": 1.0005292111554163, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:510", "inputs": [ "l:321", "l:511" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:758" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 51.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:511", "inputs": [ "l:512" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:759" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:512", "inputs": [ "l:513", "l:513", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:760" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:513", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:761" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:514", "inputs": [ "l:515" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:763" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:515", "inputs": [ "l:516", "l:520" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:764" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:516", "inputs": [ "l:517" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:765" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:517", "inputs": [ "l:518", "l:519" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:767" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:518", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "22886717dc687b1348f7927d52b5cc1dc2d3824d3a43c602bf3a48e24296a1be", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:768" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:519", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "41b28832e878ed27557031635fd40c492bfee66e307f10662e84521274fee90c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:769" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:520", "inputs": [ "l:521" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:771" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:521", "inputs": [ "l:522", "l:534", "l:540" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:772" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:522", "inputs": [ "l:523" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:773" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:523", "inputs": [ "l:34", "l:524" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:774" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:524", "inputs": [ "l:525" ], "name": "collapse_Add#898_fused_fused_add_const_multiply_const_onehot_lookup_select_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "a14f4d1390db4537b651a5a1b5b4e6ac57144981f0effccc357c5775d15f02ce", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d99d30b07223eb5feb65577e6f6b2fde883d4577d966616243495876cd1f06af", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "09511141e3c1dbec582737bc8323c4554fd8c7cc2946636288ea30de8cf24e53", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ed13abf15b052c3722a5c567169123be0df1d86bedfd9e5e39adafe492085aa", "shape": [ 14, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:775" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:525", "inputs": [ "l:526", "l:527" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:780" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:526", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:781" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:527", "inputs": [ "l:528" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:782" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:528", "inputs": [ "l:529" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:784" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:529", "inputs": [ "l:530", "l:531" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:786" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:530", "inputs": [ "l:526" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:787" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:531", "inputs": [ "l:532" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:788" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:532", "inputs": [ "l:533", "l:533", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:789" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:533", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:790" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:534", "inputs": [ "l:535" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:792" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:535", "inputs": [ "l:60", "l:536" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:793" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:536", "inputs": [ "l:537" ], "name": "collapse_pl_equals_vector_fused_fused_add_const_multiply_const_onehot_lookup_select_0_5", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "e491d258ef7d16045840f85b621ada893b0f35f02ae622c89cc012f17e89b5fa", "shape": [ 5 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b27cd9af0efbe781be2ba8a2bcfca1dfb496151846378a5d9ebca516d75d947d", "shape": [ 5, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "09511141e3c1dbec582737bc8323c4554fd8c7cc2946636288ea30de8cf24e53", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "d53a49752f4fdd093968330713c0411479789f5462a0ddfef21ff831a000b9cd", "shape": [ 5, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 5 }, "source_nodes": [ "s:794" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005019037815396516, hi=1.0005019037815397))" } ], "claimed_range": { "lower": -0.0005019037815396516, "lower_unbounded": false, "upper": 1.0005019037815397, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005019037815396516, "lower_unbounded": false, "upper": 1.0005019037815397, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:537", "inputs": [ "l:538" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:807" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:538", "inputs": [ "l:539", "l:539", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:808" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:539", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:809" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:540", "inputs": [ "l:541" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:811" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:541", "inputs": [ "l:69", "l:542" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:812" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:542", "inputs": [ "l:543" ], "name": "collapse_pl_Add#836_fused_fused_add_const_multiply_const_onehot_lookup_select_0_206", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "27ecd8d43ef3729b3c70d8fc52d57f7c663ba44cec39e041415559617817abac", "shape": [ 206 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bd1a54bf1f24b30e5c23b5d0d6fc207a15657a73c529b3e3898c768de8f32ad4", "shape": [ 206, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "583141be6edfe40ceefc0f51e4a1071c42295c53820b499aa40b1f4275435486", "shape": [ 206, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 206 }, "source_nodes": [ "s:813" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005149277173835658, hi=1.0005292111554163))" } ], "claimed_range": { "lower": -0.0005149277173835658, "lower_unbounded": false, "upper": 1.0005292111554163, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005149277173835658, "lower_unbounded": false, "upper": 1.0005292111554163, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:543", "inputs": [ "l:285", "l:544" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:818" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 51.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:544", "inputs": [ "l:545" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:819" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:545", "inputs": [ "l:546", "l:546", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "4245ea3ab317856147a0e2b3bcc47cf6cde638f5c262235ba777f4a403c34fae", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:820" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:546", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:821" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:547", "inputs": [ "l:548" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:823" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:548", "inputs": [ "l:549", "l:553" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:824" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:549", "inputs": [ "l:550" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:825" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:550", "inputs": [ "l:551", "l:552" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:827" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:551", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "583ebb6a4984ea315508e66cd81f1c0216a0719fc08f365cca73818d9552326c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:828" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:552", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "790ba56c4d91c3c94f262121ff631b25334504dc16753aed3fd7d9dc2803db41", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:829" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:553", "inputs": [ "l:554" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:831" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:554", "inputs": [ "l:555", "l:567", "l:579" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:832" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:555", "inputs": [ "l:556" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:833" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:556", "inputs": [ "l:34", "l:557" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:834" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:557", "inputs": [ "l:558" ], "name": "collapse_Add#814_fused_fused_add_const_multiply_const_onehot_lookup_select_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "a14f4d1390db4537b651a5a1b5b4e6ac57144981f0effccc357c5775d15f02ce", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d99d30b07223eb5feb65577e6f6b2fde883d4577d966616243495876cd1f06af", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "09511141e3c1dbec582737bc8323c4554fd8c7cc2946636288ea30de8cf24e53", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ed13abf15b052c3722a5c567169123be0df1d86bedfd9e5e39adafe492085aa", "shape": [ 14, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:835" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:558", "inputs": [ "l:559", "l:560" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:840" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:559", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:841" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:560", "inputs": [ "l:561" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:842" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:561", "inputs": [ "l:562" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:844" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:562", "inputs": [ "l:563", "l:564" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:846" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:563", "inputs": [ "l:559" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:847" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:564", "inputs": [ "l:565" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:848" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:565", "inputs": [ "l:566", "l:566", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:849" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:566", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:850" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:567", "inputs": [ "l:568" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:852" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:568", "inputs": [ "l:60", "l:569" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:853" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:569", "inputs": [ "l:570" ], "name": "collapse_Add#799_fused_fused_add_const_multiply_const_onehot_lookup_select_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "a14f4d1390db4537b651a5a1b5b4e6ac57144981f0effccc357c5775d15f02ce", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d99d30b07223eb5feb65577e6f6b2fde883d4577d966616243495876cd1f06af", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "09511141e3c1dbec582737bc8323c4554fd8c7cc2946636288ea30de8cf24e53", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ed13abf15b052c3722a5c567169123be0df1d86bedfd9e5e39adafe492085aa", "shape": [ 14, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:854" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:570", "inputs": [ "l:571", "l:572" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:859" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:571", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:860" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:572", "inputs": [ "l:573" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:861" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:573", "inputs": [ "l:574" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:863" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:574", "inputs": [ "l:575", "l:576" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:865" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:575", "inputs": [ "l:571" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:866" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:576", "inputs": [ "l:577" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:867" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:577", "inputs": [ "l:578", "l:578", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:868" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:578", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:869" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:579", "inputs": [ "l:580" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:871" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:580", "inputs": [ "l:69", "l:581" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:872" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:581", "inputs": [ "l:582" ], "name": "collapse_pl_Add#758_fused_fused_add_const_multiply_const_onehot_lookup_select_0_134", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "4e705c585d757a36606f6a63afbf953330905910ea83d6b28ffce0c96629c8b7", "shape": [ 134 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "0cba320c895fe5f239fd4b74d5a47ac0c4fee5f799cfc8f13aa74af1c53d4596", "shape": [ 134, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f011a372d523de15d7790b8257097a89fe241680948311d06d2d5357992cbc3d", "shape": [ 134, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 134 }, "source_nodes": [ "s:873" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005149276857477295, hi=1.0005149276843444))" } ], "claimed_range": { "lower": -0.0005149276857477295, "lower_unbounded": false, "upper": 1.0005149276843444, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005149276857477295, "lower_unbounded": false, "upper": 1.0005149276843444, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:582", "inputs": [ "l:252", "l:583" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:878" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:583", "inputs": [ "l:584" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:879" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:584", "inputs": [ "l:585", "l:585", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2f89c627454e16505ffd9b5129358e251fa7ee409a68400e1f77bed409fc51e8", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:880" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:585", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:881" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:586", "inputs": [ "l:587" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:883" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:587", "inputs": [ "l:588", "l:592" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:884" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:588", "inputs": [ "l:589" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:885" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:589", "inputs": [ "l:590", "l:591" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:887" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:590", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "418aec4f616c4d163c2123ca102e0033c51a01409d61fd37b527cb01c86d1366", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:888" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:591", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f41aacfe564dfc54191d249d1cd466a118145c3f617069e8e30958f4f57cf340", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:889" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:592", "inputs": [ "l:593" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:891" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:593", "inputs": [ "l:594", "l:606", "l:618" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:892" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:594", "inputs": [ "l:595" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:893" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:595", "inputs": [ "l:34", "l:596" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:894" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:596", "inputs": [ "l:597" ], "name": "collapse_Add#732_fused_fused_add_const_multiply_const_onehot_lookup_select_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "a14f4d1390db4537b651a5a1b5b4e6ac57144981f0effccc357c5775d15f02ce", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d99d30b07223eb5feb65577e6f6b2fde883d4577d966616243495876cd1f06af", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "09511141e3c1dbec582737bc8323c4554fd8c7cc2946636288ea30de8cf24e53", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ed13abf15b052c3722a5c567169123be0df1d86bedfd9e5e39adafe492085aa", "shape": [ 14, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:895" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:597", "inputs": [ "l:598", "l:599" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:900" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:598", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:901" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:599", "inputs": [ "l:600" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:902" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:600", "inputs": [ "l:601" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:904" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:601", "inputs": [ "l:602", "l:603" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:906" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:602", "inputs": [ "l:598" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:907" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:603", "inputs": [ "l:604" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:908" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:604", "inputs": [ "l:605", "l:605", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:909" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:605", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:910" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:606", "inputs": [ "l:607" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:912" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:607", "inputs": [ "l:60", "l:608" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:913" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:608", "inputs": [ "l:609" ], "name": "collapse_Add#717_fused_fused_add_const_multiply_const_onehot_lookup_select_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "a14f4d1390db4537b651a5a1b5b4e6ac57144981f0effccc357c5775d15f02ce", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d99d30b07223eb5feb65577e6f6b2fde883d4577d966616243495876cd1f06af", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "09511141e3c1dbec582737bc8323c4554fd8c7cc2946636288ea30de8cf24e53", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ed13abf15b052c3722a5c567169123be0df1d86bedfd9e5e39adafe492085aa", "shape": [ 14, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:914" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:609", "inputs": [ "l:610", "l:611" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:919" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:610", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:920" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:611", "inputs": [ "l:612" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:921" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:612", "inputs": [ "l:613" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:923" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:613", "inputs": [ "l:614", "l:615" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:925" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:614", "inputs": [ "l:610" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:926" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:615", "inputs": [ "l:616" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:927" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:616", "inputs": [ "l:617", "l:617", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:928" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:617", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:929" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:618", "inputs": [ "l:619" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:931" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:619", "inputs": [ "l:69", "l:620" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:932" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:620", "inputs": [ "l:621" ], "name": "collapse_Linear#701_fused_fused_add_const_multiply_const_onehot_lookup_select_0_18", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "591d8915a727b69e3eec8ab2d56beeb5e63915c9b738b90d27515d69459b81a8", "shape": [ 18 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "83e42bcee8d553e8d90cd69779129308bb75e99fb1ffabd1cbaabacc86725d74", "shape": [ 18, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "dc1f1e6e7ae3f7c1f4477f12bcc6399e3b1e5484bb08686be582dd352a9f5ff4", "shape": [ 18, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 18 }, "source_nodes": [ "s:933" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:621", "inputs": [ "l:258" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:270" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:622", "inputs": [ "l:623" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:941" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:623", "inputs": [ "l:624", "l:628" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:942" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:624", "inputs": [ "l:625" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:943" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:625", "inputs": [ "l:626", "l:627" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:945" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:626", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "3de0afcfd401c63e8410995e0ed14091535fa844c2124bc3eee71147919929ba", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:946" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:627", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "e199fd05223bf62f88cccb9b9409bfa30c64669d538c5fe324439c03e8718de9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:947" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:628", "inputs": [ "l:629" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:949" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:629", "inputs": [ "l:630", "l:633", "l:636" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:950" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:630", "inputs": [ "l:631" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:951" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:631", "inputs": [ "l:34", "l:632" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:952" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:632", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d3d4b4498b152d62ffae25a2127656848110f4397c0385d2fa75457e20fbbf30", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:953" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:633", "inputs": [ "l:634" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:954" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:634", "inputs": [ "l:60", "l:635" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:955" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:635", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d3d4b4498b152d62ffae25a2127656848110f4397c0385d2fa75457e20fbbf30", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:956" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:636", "inputs": [ "l:637" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:957" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:637", "inputs": [ "l:69", "l:638" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:958" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:638", "inputs": [ "l:639" ], "name": "collapse_pl_equals_vector_fused_fused_add_const_multiply_const_onehot_lookup_select_0_5", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "e491d258ef7d16045840f85b621ada893b0f35f02ae622c89cc012f17e89b5fa", "shape": [ 5 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b27cd9af0efbe781be2ba8a2bcfca1dfb496151846378a5d9ebca516d75d947d", "shape": [ 5, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "09511141e3c1dbec582737bc8323c4554fd8c7cc2946636288ea30de8cf24e53", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "d53a49752f4fdd093968330713c0411479789f5462a0ddfef21ff831a000b9cd", "shape": [ 5, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 5 }, "source_nodes": [ "s:959" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005019037815396516, hi=1.0005019037815397))" } ], "claimed_range": { "lower": -0.0005019037815396516, "lower_unbounded": false, "upper": 1.0005019037815397, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005019037815396516, "lower_unbounded": false, "upper": 1.0005019037815397, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:639", "inputs": [ "l:640" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:972" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:640", "inputs": [ "l:641", "l:641", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:973" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:641", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:974" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:642", "inputs": [ "l:643" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:976" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0250499999999994))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:643", "inputs": [ "l:644", "l:648" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:977" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:644", "inputs": [ "l:645" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:978" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:645", "inputs": [ "l:646", "l:647" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:980" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:646", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "52991e853857f7561b6157d0b54b4a07ae9c0e669754333e62de05736841660b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:981" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:647", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c63332a208ca2aa557eddae88226b8fc2a8d2adbff8f79f7fc3f3808bc062c9f", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:982" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:648", "inputs": [ "l:649" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:984" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:649", "inputs": [ "l:650", "l:666", "l:690" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:985" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:650", "inputs": [ "l:651" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:986" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:651", "inputs": [ "l:34", "l:652" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:987" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:652", "inputs": [ "l:653" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:988" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:653", "inputs": [ "l:654", "l:655", "l:656" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:989" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:654", "inputs": [], "name": "collapse_pl_onehot_lookup_select_compare", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ], "values": [ -1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:990" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": -1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:655", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:998" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:656", "inputs": [ "l:657", "l:661", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:999" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:657", "inputs": [ "l:658" ], "name": "collapse_pl_onehot_lookup_select_fused_add_const_multiply_const_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5e68373512ea890f3e6475101e37221c62d22733c0c0540e313605ea24a54e70", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "4238e2868b32f19b177ba68d49da3794bff61aeb9d5f5e669fdfc3026400f239", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "ccaf6f183579497e8bfcd71045c04286fd3c2e60f3641e3eea164b761a4494b7", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c3b1db5e1286da88ecbfed9ffea23c4f2258dc88f35369828e988c5e32cb44da", "shape": [ 14, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:1000" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005018967050747233, hi=1.0005018967050747))" } ], "claimed_range": { "lower": -0.0005018967050747233, "lower_unbounded": false, "upper": 1.0005018967050747, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005018967050747233, "lower_unbounded": false, "upper": 1.0005018967050747, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:658", "inputs": [ "l:659" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:994" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:659", "inputs": [ "l:660", "l:660", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:995" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:660", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:996" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:661", "inputs": [ "l:662" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1dc5c8e021c663cd8f7ecf1fb0c6d4112bc8d7f3c9e0095cd26bd7af7b8d7f13", "shape": [ 4 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9f7ce09d183259d69fb622b5248bdb85f5f8493e54529def7b337f69760ed024", "shape": [ 4, 4 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1005" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:662", "inputs": [ "l:663" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9017d307c9bd1ac42160d5fae5ff3c4ea7782dd5541b1e83341861e134cae614", "shape": [ 16 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "e4c555906ab7dff40f54eb6916d8ede133b922f31fbcd7f2d25bd787286bf7d3", "shape": [ 16, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e74125e44e9343a83c86395ec81731b98f9aefbe35b61529fe730de3c82a8aab", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e14367196378c02a631dc1b99191297e21c25fac83ce07c8ff35070293c44ccd", "shape": [ 16, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 16 }, "source_nodes": [ "s:1007" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:663", "inputs": [ "l:664", "l:665" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1008" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -6.357438532905364, "lower_unbounded": false, "upper": 24.52410519957203, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:664", "inputs": [ "l:17" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6bd5e30e99b6cfe9c9e85bcbe7ae22cda0df1fb6f5c858c4448e5c127424c7f4", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1009" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -6.357438532905364, "lower_unbounded": false, "upper": 23.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:665", "inputs": [ "l:664" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1010" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -5.357438532905364, "lower_unbounded": false, "upper": 24.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:666", "inputs": [ "l:667" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1011" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:667", "inputs": [ "l:60", "l:668" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1012" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:668", "inputs": [ "l:669" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:1013" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:669", "inputs": [ "l:670", "l:674", "l:689" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1014" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:670", "inputs": [ "l:671" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1015" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:671", "inputs": [ "l:672" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1016" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:672", "inputs": [ "l:673", "l:673", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a3eaf4f37ccad625864abeda4df5d0fce93c2f3d0f30ade108a13425a5ded31a", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1017" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:673", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1018" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:674", "inputs": [ "l:675", "l:684", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "be94494d5c98aa4e14ad64130eab9a1dc7a79a46a59d9e84d353075468c0a9fd", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "59cc5d44d3cb84eddef86ad62b93e827db0d8a54aa8a8364814b8a098a8c6168", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1019" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:675", "inputs": [ "l:676" ], "name": "collapse_pl_Add#510_fused_add_const_multiply_const_0_18", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "36c3cf15221b73c39b02e6e5fb607ae56841b5e7740b07dadefa184857c4686e", "shape": [ 18 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b5c73af0bd4196d8e582357fb81569429b7fc99ee651d14b66ee2c8b30196991", "shape": [ 18, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3c94262fba2c4b101e82c6e4b6d188372ed57c862362b571f019cdb3fee87811", "shape": [ 18, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 18 }, "source_nodes": [ "s:1020" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005009498258114535, hi=1.0004999948259936))" } ], "claimed_range": { "lower": -0.0005009498258114535, "lower_unbounded": false, "upper": 1.0004999948259936, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005009498258114535, "lower_unbounded": false, "upper": 1.0004999948259936, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:676", "inputs": [ "l:677", "l:683" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1024" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -5.00435101171875, "lower_unbounded": false, "upper": 7.339859852864584, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:677", "inputs": [ "l:678" ], "name": "fused_min_add_const", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "source_nodes": [ "s:1025" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.0021755078125, "lower_unbounded": false, "upper": 7.337684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:678", "inputs": [ "l:679", "l:682" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1027" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:679", "inputs": [ "l:680" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1028" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:680", "inputs": [ "l:681", "l:681", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1029" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:681", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1030" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:682", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d88c86f15bbea365d658ad95a81d45367c465f7af6f7264fb077f01747ddc77d", "shape": [ 1 ], "values": [ 2.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1031" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 2.0, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:683", "inputs": [ "l:671" ], "name": "collapse_pl_onehot_lookup_select_fused_fused_add_const_multiply_const_negate_0_6", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "fe754c675d8bf9f8783d6e60cfb17700fb404b30f3baff020938d5481b6b3f03", "shape": [ 6 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "4bd660c51a6227a2beb08c8494b2abdc0428ea57785218bb61d4b2e5930c822c", "shape": [ 6, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e2eb7c48e6c80be887debb7ac462fbfa63dc60911e430c9fecbabde0de67f8fd", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 6 }, "source_nodes": [ "s:1032" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=0.0005007145165952182))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 0.0005007145165952182, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 0.0005007145165952182, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:684", "inputs": [ "l:685" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "fbfe4ca61ec299680879894a16ca776f81f23454312659ac8736249aa1e9722b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1037" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:685", "inputs": [ "l:686" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "source_nodes": [ "s:1039" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:686", "inputs": [ "l:687", "l:688" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1040" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -8.357438532905364, "lower_unbounded": false, "upper": 22.52410519957203, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:687", "inputs": [ "l:17" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "053ac09e0109c6d84088a0106d301d00ebe129455cb7889c911000d6f462271f", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1041" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -8.357438532905364, "lower_unbounded": false, "upper": 21.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:688", "inputs": [ "l:687" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1042" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -7.357438532905364, "lower_unbounded": false, "upper": 22.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:689", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "44250fce6a5c41d7a7e24f8100dbb6befdf5b44ab7d94fe23f9cec9ea6693f46", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1043" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:690", "inputs": [ "l:691" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1044" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:691", "inputs": [ "l:69", "l:692" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1045" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:692", "inputs": [ "l:693" ], "name": "collapse_Add#505_fused_fused_add_const_multiply_const_onehot_lookup_select_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "a14f4d1390db4537b651a5a1b5b4e6ac57144981f0effccc357c5775d15f02ce", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d99d30b07223eb5feb65577e6f6b2fde883d4577d966616243495876cd1f06af", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "09511141e3c1dbec582737bc8323c4554fd8c7cc2946636288ea30de8cf24e53", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ed13abf15b052c3722a5c567169123be0df1d86bedfd9e5e39adafe492085aa", "shape": [ 14, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:1046" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:693", "inputs": [ "l:694", "l:695" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1051" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:694", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1052" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:695", "inputs": [ "l:696" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1053" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:696", "inputs": [ "l:697" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1055" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:697", "inputs": [ "l:698", "l:699" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1057" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:698", "inputs": [ "l:694" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1058" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:699", "inputs": [ "l:700" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:1059" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:700", "inputs": [ "l:701", "l:701", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1060" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:701", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1061" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:702", "inputs": [ "l:703" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1063" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0250499999999994))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:703", "inputs": [ "l:704", "l:708" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1064" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:704", "inputs": [ "l:705" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1065" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:705", "inputs": [ "l:706", "l:707" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1067" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:706", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ec72bdc48ad6906fca6435b50f45617abf02e58e2863796f068e2d57eca6f101", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1068" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:707", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "d35180f5b6f03da9db760a267ff605e9cc819de41520fc22aacc937ca0e23463", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1069" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:708", "inputs": [ "l:709" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1071" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:709", "inputs": [ "l:710", "l:720", "l:739" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1072" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:710", "inputs": [ "l:711" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1073" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:711", "inputs": [ "l:34", "l:712" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1074" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:712", "inputs": [ "l:713" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:1075" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:713", "inputs": [ "l:714", "l:655", "l:718" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1076" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:714", "inputs": [ "l:715" ], "name": "collapse_pl_onehot_lookup_select_compare_0_6", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "487e42631681a02a793e496ab694e8c775f2a57a6192ccf199db39c8884aa917", "shape": [ 6 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b89272f1a6c0413461abab168045ef49fbf9fb729dad42f0654f0dfd42739226", "shape": [ 6, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9abbdc8b936921abf739ee2c89ed6b5934597474c7daab7f2374a992ddd9abe1", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 6 }, "source_nodes": [ "s:1077" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=1.0005037927840583))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005037927840583, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005037927840583, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:715", "inputs": [ "l:716" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1081" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:716", "inputs": [ "l:717", "l:717", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1082" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:717", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1083" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:718", "inputs": [ "l:719", "l:661", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1085" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:719", "inputs": [ "l:715" ], "name": "collapse_pl_onehot_lookup_select_fused_add_const_multiply_const_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5e68373512ea890f3e6475101e37221c62d22733c0c0540e313605ea24a54e70", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "4238e2868b32f19b177ba68d49da3794bff61aeb9d5f5e669fdfc3026400f239", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "6855960ee428276148fc8745625c44bde7ae835a9e2fa181cc917af9bcad0d6a", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e3659df1ea7f33dc63395fd224b86ef379d50d7fb77b0ce2b9e22b40e7d9bd8a", "shape": [ 14, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:1086" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005018967050747233, hi=1.000501824564918))" } ], "claimed_range": { "lower": -0.0005018967050747233, "lower_unbounded": false, "upper": 1.000501824564918, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005018967050747233, "lower_unbounded": false, "upper": 1.000501824564918, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:720", "inputs": [ "l:721" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1091" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:721", "inputs": [ "l:60", "l:722" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1092" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:722", "inputs": [ "l:723" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:1093" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:723", "inputs": [ "l:724", "l:736", "l:737" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1094" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:724", "inputs": [ "l:725" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1095" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:725", "inputs": [ "l:726", "l:732" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1096" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.00435101171875, "lower_unbounded": false, "upper": 8.339859852864583, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:726", "inputs": [ "l:727" ], "name": "fused_min_add_const", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "source_nodes": [ "s:1097" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0021755078125, "lower_unbounded": false, "upper": 8.337684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:727", "inputs": [ "l:728", "l:731" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1099" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:728", "inputs": [ "l:729" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1100" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:729", "inputs": [ "l:730", "l:730", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1101" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:730", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1102" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:731", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d88c86f15bbea365d658ad95a81d45367c465f7af6f7264fb077f01747ddc77d", "shape": [ 1 ], "values": [ 2.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1103" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 2.0, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:732", "inputs": [ "l:733" ], "name": "collapse_pl_onehot_lookup_select_fused_fused_add_const_multiply_const_negate_0_6", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "fe754c675d8bf9f8783d6e60cfb17700fb404b30f3baff020938d5481b6b3f03", "shape": [ 6 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "4bd660c51a6227a2beb08c8494b2abdc0428ea57785218bb61d4b2e5930c822c", "shape": [ 6, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e2eb7c48e6c80be887debb7ac462fbfa63dc60911e430c9fecbabde0de67f8fd", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 6 }, "source_nodes": [ "s:1104" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=0.0005007145165952182))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 0.0005007145165952182, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 0.0005007145165952182, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:733", "inputs": [ "l:734" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1109" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:734", "inputs": [ "l:735", "l:735", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "64b0a4812c027fb37ce2b91630d9df5c6549ab2445a912d1b8ded79071f00141", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1110" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:735", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1111" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:736", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1112" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:737", "inputs": [ "l:738", "l:684", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "be94494d5c98aa4e14ad64130eab9a1dc7a79a46a59d9e84d353075468c0a9fd", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "59cc5d44d3cb84eddef86ad62b93e827db0d8a54aa8a8364814b8a098a8c6168", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1113" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:738", "inputs": [ "l:725" ], "name": "collapse_pl_Add#450_fused_add_const_multiply_const_0_18", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "556db906ed73c27f6070d6a5d3ee92a44b8528dede640294180ac8f8e3411b57", "shape": [ 18 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b5c73af0bd4196d8e582357fb81569429b7fc99ee651d14b66ee2c8b30196991", "shape": [ 18, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "df9f4677f31a068b62480a237fac28c255f0c5e2e2e6e46ede816df17c89585a", "shape": [ 18, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 18 }, "source_nodes": [ "s:1114" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005009504848178569, hi=1.0004999948259936))" } ], "claimed_range": { "lower": -0.0005009504848178569, "lower_unbounded": false, "upper": 1.0004999948259936, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005009504848178569, "lower_unbounded": false, "upper": 1.0004999948259936, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:739", "inputs": [ "l:740" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1119" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:740", "inputs": [ "l:69", "l:741" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1120" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:741", "inputs": [ "l:742" ], "name": "collapse_Add#443_fused_fused_add_const_multiply_const_onehot_lookup_select_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "a14f4d1390db4537b651a5a1b5b4e6ac57144981f0effccc357c5775d15f02ce", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d99d30b07223eb5feb65577e6f6b2fde883d4577d966616243495876cd1f06af", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "09511141e3c1dbec582737bc8323c4554fd8c7cc2946636288ea30de8cf24e53", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ed13abf15b052c3722a5c567169123be0df1d86bedfd9e5e39adafe492085aa", "shape": [ 14, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:1121" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:742", "inputs": [ "l:743", "l:744" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1126" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:743", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1127" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:744", "inputs": [ "l:745" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1128" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:745", "inputs": [ "l:746" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1130" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:746", "inputs": [ "l:747", "l:748" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1132" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:747", "inputs": [ "l:743" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1133" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:748", "inputs": [ "l:749" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:1134" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:749", "inputs": [ "l:750", "l:750", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1135" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:750", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1136" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:751", "inputs": [ "l:752" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1138" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0250499999999994))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:752", "inputs": [ "l:753", "l:757" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1139" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:753", "inputs": [ "l:754" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1140" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:754", "inputs": [ "l:755", "l:756" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1142" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:755", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f168e05b01b0f8f0195051bd5bbc1643e3448dcb28d39baa7f9ee0dc6482cb03", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1143" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:756", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c152adf65b2187fcc340d0af789d32925fae822758408a76032143d93dcf2f", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1144" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:757", "inputs": [ "l:758" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1146" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:758", "inputs": [ "l:759", "l:769", "l:787" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1147" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:759", "inputs": [ "l:760" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1148" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:760", "inputs": [ "l:34", "l:761" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1149" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:761", "inputs": [ "l:762" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:1150" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:762", "inputs": [ "l:763", "l:655", "l:767" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1151" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:763", "inputs": [ "l:764" ], "name": "collapse_pl_onehot_lookup_select_compare_0_6", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25f2b753eff2ffc8ca116bdfd0ce36940797a3e0517520b04559a24685a69052", "shape": [ 6 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "575988e98b61c9f426c7c9d47b991384ebade7cb91b2388cb9e0775fcb678ea8", "shape": [ 6, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9e80fff6d1d9e7dac156976cdc80e046a972ca68dc9ce6bb88f71e498ae771e3", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 6 }, "source_nodes": [ "s:1152" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=1.0005037977642302))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005037977642302, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005037977642302, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:764", "inputs": [ "l:765" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1156" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:765", "inputs": [ "l:766", "l:766", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1157" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:766", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1158" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:767", "inputs": [ "l:768", "l:661", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1160" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:768", "inputs": [ "l:764" ], "name": "collapse_pl_onehot_lookup_select_fused_add_const_multiply_const_0_9", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "74e47b840c13b3151abcc24f30275883a9d0c179f03883dff826c00cfe4429a5", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "341ee570fd5adc79e7f35fa7160781aa82ca4e6594b5d4ffbc5fdf6dccc86637", "shape": [ 9, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7472a1a3ade1c12f2ffe2837ae401d69022dc0504bcf8274b4f49b9ad8b248c4", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3fc0cf8600c41337351ff8e30b319401b5967a3e7e485a32ba6068b35999f5bb", "shape": [ 9, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:1161" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005018990702600159, hi=1.0005))" } ], "claimed_range": { "lower": -0.0005018990702600159, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005018990702600159, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:769", "inputs": [ "l:770" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1166" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:770", "inputs": [ "l:60", "l:771" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1167" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:771", "inputs": [ "l:772" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:1168" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:772", "inputs": [ "l:773", "l:736", "l:785" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1169" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:773", "inputs": [ "l:774" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1170" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:774", "inputs": [ "l:775", "l:781" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1171" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0043510117187497, "lower_unbounded": false, "upper": 9.339859852864583, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:775", "inputs": [ "l:776" ], "name": "fused_min_add_const", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "d88c86f15bbea365d658ad95a81d45367c465f7af6f7264fb077f01747ddc77d", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "source_nodes": [ "s:1172" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0021755078125, "lower_unbounded": false, "upper": 9.337684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:776", "inputs": [ "l:777", "l:780" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1174" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:777", "inputs": [ "l:778" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1175" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:778", "inputs": [ "l:779", "l:779", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1176" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:779", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1177" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:780", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d88c86f15bbea365d658ad95a81d45367c465f7af6f7264fb077f01747ddc77d", "shape": [ 1 ], "values": [ 2.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1178" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 2.0, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:781", "inputs": [ "l:782" ], "name": "collapse_pl_onehot_lookup_select_fused_fused_add_const_multiply_const_negate_0_6", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "fe754c675d8bf9f8783d6e60cfb17700fb404b30f3baff020938d5481b6b3f03", "shape": [ 6 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "4bd660c51a6227a2beb08c8494b2abdc0428ea57785218bb61d4b2e5930c822c", "shape": [ 6, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e2eb7c48e6c80be887debb7ac462fbfa63dc60911e430c9fecbabde0de67f8fd", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 6 }, "source_nodes": [ "s:1179" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=0.0005007145165952182))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 0.0005007145165952182, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 0.0005007145165952182, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:782", "inputs": [ "l:783" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1184" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:783", "inputs": [ "l:784", "l:784", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "b21306b2ca4e61b9d9672dee3108ec3e685bfa6056de62cdb01cb85dc62821e9", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1185" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:784", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1186" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:785", "inputs": [ "l:786", "l:684", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "be94494d5c98aa4e14ad64130eab9a1dc7a79a46a59d9e84d353075468c0a9fd", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "59cc5d44d3cb84eddef86ad62b93e827db0d8a54aa8a8364814b8a098a8c6168", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1187" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:786", "inputs": [ "l:774" ], "name": "collapse_pl_Add#391_fused_add_const_multiply_const_0_18", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c08b7df60941324febf490b8f2387146ab1caacd585ec47ae4f4e4d6d6bedcac", "shape": [ 18 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b5c73af0bd4196d8e582357fb81569429b7fc99ee651d14b66ee2c8b30196991", "shape": [ 18, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "385aed2741e5c35a305d84825e41e24dac5979de5cad3756f7764046b67f5279", "shape": [ 18, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 18 }, "source_nodes": [ "s:1188" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005009509511283776, hi=1.0004999948259936))" } ], "claimed_range": { "lower": -0.0005009509511283776, "lower_unbounded": false, "upper": 1.0004999948259936, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005009509511283776, "lower_unbounded": false, "upper": 1.0004999948259936, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:787", "inputs": [ "l:788" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1193" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:788", "inputs": [ "l:69", "l:789" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1194" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:789", "inputs": [ "l:790" ], "name": "collapse_Add#381_fused_fused_add_const_multiply_const_onehot_lookup_select_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "a14f4d1390db4537b651a5a1b5b4e6ac57144981f0effccc357c5775d15f02ce", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d99d30b07223eb5feb65577e6f6b2fde883d4577d966616243495876cd1f06af", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "09511141e3c1dbec582737bc8323c4554fd8c7cc2946636288ea30de8cf24e53", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ed13abf15b052c3722a5c567169123be0df1d86bedfd9e5e39adafe492085aa", "shape": [ 14, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:1195" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:790", "inputs": [ "l:791", "l:792" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1200" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:791", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1201" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:792", "inputs": [ "l:793" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1202" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:793", "inputs": [ "l:794" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1204" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:794", "inputs": [ "l:795", "l:796" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1206" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:795", "inputs": [ "l:791" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1207" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:796", "inputs": [ "l:797" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:1208" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:797", "inputs": [ "l:798", "l:798", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1209" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:798", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1210" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:799", "inputs": [ "l:800" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1212" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0250499999999994))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:800", "inputs": [ "l:801", "l:805" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1213" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:801", "inputs": [ "l:802" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1214" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:802", "inputs": [ "l:803", "l:804" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1216" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:803", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "cb6b0c11488519a6658b9e429b5a8ceedfd51a949605930741312ea58bbe797b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1217" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:804", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "106bb444492286cece2c2f3471615ce3ed37466f891bf2df4c164be23d396f57", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1218" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:805", "inputs": [ "l:806" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1220" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:806", "inputs": [ "l:807", "l:817", "l:835" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1221" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:807", "inputs": [ "l:808" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1222" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:808", "inputs": [ "l:34", "l:809" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1223" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:809", "inputs": [ "l:810" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:1224" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:810", "inputs": [ "l:811", "l:655", "l:815" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1225" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:811", "inputs": [ "l:812" ], "name": "collapse_pl_onehot_lookup_select_compare_0_4", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b1caea877d36418e901563a4714ea5c28e7fb7990e12b7df51126eb73b475dab", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bfcb9e968d903d82e6dcf8c6cad83e9f50a44062483b845083a44f4bbcc645cb", "shape": [ 4, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "943f6d14e14382442b5e35e9f77fbb35926306d9cdddec1abc2ddb5f5e4e5fda", "shape": [ 4, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 4 }, "source_nodes": [ "s:1226" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=1.0005))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:812", "inputs": [ "l:813" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1230" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:813", "inputs": [ "l:814", "l:814", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "4a219c06af2467623290b375e580c952ecdae33edf14aa594ac393d27e76b67e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1231" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:814", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1232" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:815", "inputs": [ "l:816", "l:661", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1234" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:816", "inputs": [ "l:812" ], "name": "collapse_pl_onehot_lookup_select_fused_add_const_multiply_const_0_4", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f7768fa2f414969ac92fa7c2ef9efcfd201c4653ad34bf8306ea7c4329f2b829", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "105e6908375dfb2747833fbb73c90b2496566a9d4868858df7ca74a3ce68b04b", "shape": [ 4, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "166c3436231871f810594173ceec8bcc6d43229b315edc758eb096fa5407cd94", "shape": [ 4, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 4 }, "source_nodes": [ "s:1235" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005, hi=1.0005))" } ], "claimed_range": { "lower": -0.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:817", "inputs": [ "l:818" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1240" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:818", "inputs": [ "l:60", "l:819" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1241" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:819", "inputs": [ "l:820" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:1242" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:820", "inputs": [ "l:821", "l:736", "l:833" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1243" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:821", "inputs": [ "l:822" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1244" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:822", "inputs": [ "l:823", "l:829" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1245" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0043510117187497, "lower_unbounded": false, "upper": 10.339859852864583, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:823", "inputs": [ "l:824" ], "name": "fused_min_add_const", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "ea2845900b5856c9bf354b1aa9761b5aa6888e5ed61738fe9579ca42bc0f6054", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "source_nodes": [ "s:1246" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755078124999, "lower_unbounded": false, "upper": 10.337684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:824", "inputs": [ "l:825", "l:828" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1248" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:825", "inputs": [ "l:826" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1249" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:826", "inputs": [ "l:827", "l:827", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "4a219c06af2467623290b375e580c952ecdae33edf14aa594ac393d27e76b67e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1250" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:827", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1251" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:828", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d88c86f15bbea365d658ad95a81d45367c465f7af6f7264fb077f01747ddc77d", "shape": [ 1 ], "values": [ 2.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1252" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 2.0, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:829", "inputs": [ "l:830" ], "name": "collapse_pl_onehot_lookup_select_fused_fused_add_const_multiply_const_negate_0_6", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "fe754c675d8bf9f8783d6e60cfb17700fb404b30f3baff020938d5481b6b3f03", "shape": [ 6 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "4bd660c51a6227a2beb08c8494b2abdc0428ea57785218bb61d4b2e5930c822c", "shape": [ 6, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e2eb7c48e6c80be887debb7ac462fbfa63dc60911e430c9fecbabde0de67f8fd", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 6 }, "source_nodes": [ "s:1253" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=0.0005007145165952182))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 0.0005007145165952182, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 0.0005007145165952182, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:830", "inputs": [ "l:831" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1258" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:831", "inputs": [ "l:832", "l:832", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "c570eb9ab41f73b36bf50b3fd1503550ce8d9bc28113f355f45e5df3f5faa9b3", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1259" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:832", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1260" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:833", "inputs": [ "l:834", "l:684", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "be94494d5c98aa4e14ad64130eab9a1dc7a79a46a59d9e84d353075468c0a9fd", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "59cc5d44d3cb84eddef86ad62b93e827db0d8a54aa8a8364814b8a098a8c6168", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1261" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:834", "inputs": [ "l:822" ], "name": "collapse_pl_Add#325_fused_add_const_multiply_const_0_18", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "3c49796b4e1a00e481f7a3a31f0c768d59c8275f999e9d318166f261cac8d94f", "shape": [ 18 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b5c73af0bd4196d8e582357fb81569429b7fc99ee651d14b66ee2c8b30196991", "shape": [ 18, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e38c49a15bff52c0dbf257e35040015e702e478627964ee140fee5636defccf0", "shape": [ 18, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 18 }, "source_nodes": [ "s:1262" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005009512984800043, hi=1.0004999948259936))" } ], "claimed_range": { "lower": -0.0005009512984800043, "lower_unbounded": false, "upper": 1.0004999948259936, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005009512984800043, "lower_unbounded": false, "upper": 1.0004999948259936, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:835", "inputs": [ "l:836" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1267" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:836", "inputs": [ "l:69", "l:837" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1268" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:837", "inputs": [ "l:838" ], "name": "collapse_Add#307_fused_fused_add_const_multiply_const_onehot_lookup_select_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "a14f4d1390db4537b651a5a1b5b4e6ac57144981f0effccc357c5775d15f02ce", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d99d30b07223eb5feb65577e6f6b2fde883d4577d966616243495876cd1f06af", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "09511141e3c1dbec582737bc8323c4554fd8c7cc2946636288ea30de8cf24e53", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ed13abf15b052c3722a5c567169123be0df1d86bedfd9e5e39adafe492085aa", "shape": [ 14, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:1269" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:838", "inputs": [ "l:839", "l:840" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1274" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:839", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1275" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:840", "inputs": [ "l:841" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1276" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:841", "inputs": [ "l:842" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1278" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:842", "inputs": [ "l:843", "l:844" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1280" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:843", "inputs": [ "l:839" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1281" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:844", "inputs": [ "l:845" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:1282" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:845", "inputs": [ "l:846", "l:846", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1283" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:846", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1284" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:847", "inputs": [ "l:848" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1286" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:848", "inputs": [ "l:849", "l:853" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1287" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:849", "inputs": [ "l:850" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1288" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:850", "inputs": [ "l:851", "l:852" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1290" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:851", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "79917cb2415012d945ab2169ee0312f15acd06846b035207da169eb0ce402674", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1291" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:852", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c685308f2ab536a2889ee91d69f78db27e7708c0ae08ee9452a5f58d5395e79b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1292" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:853", "inputs": [ "l:854" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1294" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:854", "inputs": [ "l:855", "l:858", "l:861" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1295" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:855", "inputs": [ "l:856" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1296" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:856", "inputs": [ "l:34", "l:857" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1297" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:857", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1298" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:858", "inputs": [ "l:859" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1299" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:859", "inputs": [ "l:60", "l:860" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1300" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:860", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1301" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:861", "inputs": [ "l:862" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1302" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:862", "inputs": [ "l:69", "l:863" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1303" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:863", "inputs": [ "l:864" ], "name": "collapse_Add#278_fused_fused_add_const_multiply_const_onehot_lookup_select_0_14", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "a14f4d1390db4537b651a5a1b5b4e6ac57144981f0effccc357c5775d15f02ce", "shape": [ 14 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d99d30b07223eb5feb65577e6f6b2fde883d4577d966616243495876cd1f06af", "shape": [ 14, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "09511141e3c1dbec582737bc8323c4554fd8c7cc2946636288ea30de8cf24e53", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ed13abf15b052c3722a5c567169123be0df1d86bedfd9e5e39adafe492085aa", "shape": [ 14, 33 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 14 }, "source_nodes": [ "s:1304" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.002175503906250008, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906250008, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:864", "inputs": [ "l:865", "l:866" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1309" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:865", "inputs": [ "l:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1310" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:866", "inputs": [ "l:867" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1311" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:867", "inputs": [ "l:868" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1313" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:868", "inputs": [ "l:869", "l:870" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1315" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:869", "inputs": [ "l:865" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "db8d1a556ec9527f63930661298f4b8607cfcecb3c364235751acfef1ec8ed1a", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1316" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:870", "inputs": [ "l:871" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:1317" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:871", "inputs": [ "l:872", "l:872", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1318" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:872", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1319" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:873", "inputs": [ "l:874" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1321" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:874", "inputs": [ "l:875", "l:879" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1322" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:875", "inputs": [ "l:876" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1323" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:876", "inputs": [ "l:877", "l:878" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1325" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:877", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "06880e085843c937418cc66e843c4da6c39a4617eaea1521f400b8f849e67dfa", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1326" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:878", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "78ce7ad60acf4307970c9a59f23a0d670b828dd102abe6c47a1ed75fbbb679f6", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1327" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:879", "inputs": [ "l:880" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1329" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:880", "inputs": [ "l:881", "l:884", "l:887" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1330" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:881", "inputs": [ "l:882" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1331" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:882", "inputs": [ "l:34", "l:883" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1332" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:883", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1333" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:884", "inputs": [ "l:885" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1334" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:885", "inputs": [ "l:60", "l:886" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1335" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:886", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1336" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:887", "inputs": [ "l:888" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1337" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:888", "inputs": [ "l:69", "l:889" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1338" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:889", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d3d4b4498b152d62ffae25a2127656848110f4397c0385d2fa75457e20fbbf30", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1339" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:890", "inputs": [ "l:891" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1340" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0200249999999995))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:891", "inputs": [ "l:892", "l:896" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1341" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:892", "inputs": [ "l:893" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1342" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:893", "inputs": [ "l:894", "l:895" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1344" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:894", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0d35a65ae9d33e342992c713cdfe6e51a88a3ccca57c25d7003ecaef1a060be9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1345" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:895", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5eab47f016015070cba2c5ceb2d8bececfdf333d0f25fc7ab6c73cb8201cf7cc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1346" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:896", "inputs": [ "l:897" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1348" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:897", "inputs": [ "l:898", "l:901", "l:904" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1349" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:898", "inputs": [ "l:899" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1350" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:899", "inputs": [ "l:34", "l:900" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1351" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:900", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1352" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:901", "inputs": [ "l:902" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1353" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:902", "inputs": [ "l:60", "l:903" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1354" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:903", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1355" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:904", "inputs": [ "l:905" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1356" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:905", "inputs": [ "l:69", "l:906" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1357" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:906", "inputs": [ "l:907" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:1358" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:907", "inputs": [ "l:908", "l:909", "l:910" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1359" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:908", "inputs": [], "name": "collapse_pl_onehot_lookup_select_compare", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ], "values": [ -1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1360" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": -1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:909", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1368" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:910", "inputs": [ "l:911", "l:915", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "0ebfee506020ed8b5565a2f5252bea8bae295626330b537d4c9b488a46a712de", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "9010fceb49e61304a784d8dc3d60e4881a4749caa665c18a91caecfad9bbce83", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1369" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:911", "inputs": [ "l:912" ], "name": "collapse_pl_onehot_lookup_select_fused_add_const_multiply_const_0_22", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "71d72fb2f7badab0e55817a08e4bb72b070273af6c827db942ce68113c32fcea", "shape": [ 22 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2034aba25353c5ea9c72c30ae13f6f3e6eb15a7c7647604695f31d6f606a15c8", "shape": [ 22, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "76d8f3f98a45e3ee1027516634ec6b89daf1f03943a55f0e2e4f205518614fa2", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f97212a6620456cae5cf87903e67cf19e5307de4e1470e4bc338ced23576ea54", "shape": [ 22, 6 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 22 }, "source_nodes": [ "s:1370" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005037650279093929, hi=1.0005037650279094))" } ], "claimed_range": { "lower": -0.0005037650279093929, "lower_unbounded": false, "upper": 1.0005037650279094, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005037650279093929, "lower_unbounded": false, "upper": 1.0005037650279094, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:912", "inputs": [ "l:913" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1364" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:913", "inputs": [ "l:914", "l:914", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1365" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:914", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1366" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:915", "inputs": [ "l:916" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "9ba54d57656313e94dc021212d7e07524183ae6401113a0eac079e75d7301d33", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f1af9fe3a978a5ad01515b76da2145665eca0ffdd72ab3b8e526c05b1dd8d2f5", "shape": [ 6, 6 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1375" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:916", "inputs": [ "l:917" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1fa343150bb83ed9689c44694dc284a239460221edd24178bbe9c0642874e4cf", "shape": [ 24 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "38c349087c69fb3195cc6f14b0efaee2cbddf46673bcf566cbd455c6fc27569f", "shape": [ 24, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "90ba88e2e295f2a5205122dcf0b0d15b5e9bd0ee828ef8e3e4ddf3c6bc53a320", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "062c0d53503ab27e308d67f283dcb28c04de7a7d7e721bf62691c3f373150efb", "shape": [ 24, 6 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 24 }, "source_nodes": [ "s:1377" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:917", "inputs": [ "l:918", "l:919" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1378" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -8.357438532905364, "lower_unbounded": false, "upper": 22.52410519957203, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:918", "inputs": [ "l:17" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "053ac09e0109c6d84088a0106d301d00ebe129455cb7889c911000d6f462271f", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1379" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -8.357438532905364, "lower_unbounded": false, "upper": 21.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:919", "inputs": [ "l:918" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1380" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -7.357438532905364, "lower_unbounded": false, "upper": 22.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:920", "inputs": [ "l:921" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1381" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0200249999999995))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:921", "inputs": [ "l:922", "l:926" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1382" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:922", "inputs": [ "l:923" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1383" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:923", "inputs": [ "l:924", "l:925" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1385" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:924", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "d484ed60a29fdabc7a9e604d97a1de9b86fb26900f9f27e08f9e560d463c68b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1386" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:925", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6537e8db32fb220fff8c4e77ee94d0ed69757893ff2f79d6442ed507ed42042c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1387" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:926", "inputs": [ "l:927" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1389" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:927", "inputs": [ "l:928", "l:931", "l:934" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1390" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:928", "inputs": [ "l:929" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1391" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:929", "inputs": [ "l:34", "l:930" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1392" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:930", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1393" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:931", "inputs": [ "l:932" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1394" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:932", "inputs": [ "l:60", "l:933" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1395" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:933", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1396" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:934", "inputs": [ "l:935" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1397" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:935", "inputs": [ "l:69", "l:936" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1398" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:936", "inputs": [ "l:937" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:1399" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:937", "inputs": [ "l:938", "l:909", "l:942" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1400" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:938", "inputs": [ "l:939" ], "name": "collapse_pl_onehot_lookup_select_compare_0_6", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "fb80eeae1a4f766597aeb24e68dc86360a2375977be70c338cacb7408dcde7ce", "shape": [ 6 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dac32ca557a204243aed62969365f5d6629b5dc60747bf3692f68aab6b3cb4e", "shape": [ 6, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f7035fa178db7c9afb42d3a245c42cefdef91bf4cf813759a634337ce506517a", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 6 }, "source_nodes": [ "s:1401" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=1.0005075229601987))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005075229601987, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005075229601987, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:939", "inputs": [ "l:940" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1405" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:940", "inputs": [ "l:941", "l:941", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1406" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:941", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1407" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:942", "inputs": [ "l:943", "l:915", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "0ebfee506020ed8b5565a2f5252bea8bae295626330b537d4c9b488a46a712de", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "9010fceb49e61304a784d8dc3d60e4881a4749caa665c18a91caecfad9bbce83", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1409" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:943", "inputs": [ "l:939" ], "name": "collapse_pl_onehot_lookup_select_fused_add_const_multiply_const_0_22", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "71d72fb2f7badab0e55817a08e4bb72b070273af6c827db942ce68113c32fcea", "shape": [ 22 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2034aba25353c5ea9c72c30ae13f6f3e6eb15a7c7647604695f31d6f606a15c8", "shape": [ 22, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "096266590f95f15a19bc5bd995e243f4dabcb9ac9843cb830b255b68a6981417", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "b17590176f000a343646f2949551a5526170fa6caccbae9386ab1888b1613c6c", "shape": [ 22, 6 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 22 }, "source_nodes": [ "s:1410" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005037650279093929, hi=1.0005017831738185))" } ], "claimed_range": { "lower": -0.0005037650279093929, "lower_unbounded": false, "upper": 1.0005017831738185, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005037650279093929, "lower_unbounded": false, "upper": 1.0005017831738185, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:944", "inputs": [ "l:945" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1415" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0200249999999995))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:945", "inputs": [ "l:946", "l:950" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1416" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:946", "inputs": [ "l:947" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1417" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:947", "inputs": [ "l:948", "l:949" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1419" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:948", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5b5d5ecc535fb6f3f2089b1ebe1411a7aee7a3b83c453733fd6335b0fa0cdbc7", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1420" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:949", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "cb24e6346737ac5db8a9f0fd7ac795a0a2dcdb2d3606e790469a23c68df90eab", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1421" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:950", "inputs": [ "l:951" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1423" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:951", "inputs": [ "l:952", "l:955", "l:958" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1424" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:952", "inputs": [ "l:953" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1425" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:953", "inputs": [ "l:34", "l:954" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1426" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:954", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1427" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:955", "inputs": [ "l:956" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1428" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:956", "inputs": [ "l:60", "l:957" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1429" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:957", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1430" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:958", "inputs": [ "l:959" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1431" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:959", "inputs": [ "l:69", "l:960" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1432" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:960", "inputs": [ "l:961" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:1433" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:961", "inputs": [ "l:962", "l:909", "l:966" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1434" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:962", "inputs": [ "l:963" ], "name": "collapse_pl_onehot_lookup_select_compare_0_6", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1e47cb2d8778cd0e83a2f6c2538429ffb61849e6812566fe4d766c556fae4a78", "shape": [ 6 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "04069aaf5e579bc025c702476fdbfa3a61e81f675ecaad8f12ead2ca3ca8b088", "shape": [ 6, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6b6d5d09952c41f5778dec23ae715cc02e03f56f11a7e72ad0a86b5ab3fc1029", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 6 }, "source_nodes": [ "s:1435" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=1.0005037836535506))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005037836535506, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005037836535506, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:963", "inputs": [ "l:964" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1439" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:964", "inputs": [ "l:965", "l:965", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1440" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:965", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1441" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:966", "inputs": [ "l:967", "l:915", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "0ebfee506020ed8b5565a2f5252bea8bae295626330b537d4c9b488a46a712de", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "9010fceb49e61304a784d8dc3d60e4881a4749caa665c18a91caecfad9bbce83", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1443" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:967", "inputs": [ "l:963" ], "name": "collapse_pl_onehot_lookup_select_fused_add_const_multiply_const_0_17", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "58181f6bc0e28e069720cd41df3f52dd0bf4ea25392087d649a28d4711345bd1", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3a292aff8d6f8a25942b86cb707d95837aa0ca764c85ea8988b0bcbd08b42b29", "shape": [ 17, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a23a041f32ad6eb94caa4b08343959a7f2d49058d97548b5c8198258f7a40317", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2d5b7e26a893753e9d78a0bfe88f368b65866734e830606c38ad1d6a11e92c81", "shape": [ 17, 6 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 17 }, "source_nodes": [ "s:1444" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005018924476506755, hi=1.0005017831738185))" } ], "claimed_range": { "lower": -0.0005018924476506755, "lower_unbounded": false, "upper": 1.0005017831738185, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005018924476506755, "lower_unbounded": false, "upper": 1.0005017831738185, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:968", "inputs": [ "l:969" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1449" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0200249999999995))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:969", "inputs": [ "l:970", "l:974" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1450" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:970", "inputs": [ "l:971" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1451" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:971", "inputs": [ "l:972", "l:973" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1453" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:972", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "55c549ac2b96d8954127159a714c5c16d1c42d0fa07f2b1c558bbe5d5fae3108", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1454" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:973", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c359380011b0a1c803214db4688b883faec47605ccd8371302b87b11efd33ee3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1455" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:974", "inputs": [ "l:975" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1457" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:975", "inputs": [ "l:976", "l:979", "l:982" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1458" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:976", "inputs": [ "l:977" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1459" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:977", "inputs": [ "l:34", "l:978" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1460" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:978", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1461" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:979", "inputs": [ "l:980" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1462" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:980", "inputs": [ "l:60", "l:981" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1463" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:981", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1464" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:982", "inputs": [ "l:983" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1465" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:983", "inputs": [ "l:69", "l:984" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1466" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:984", "inputs": [ "l:985" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:1467" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:985", "inputs": [ "l:986", "l:909", "l:990" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1468" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:986", "inputs": [ "l:987" ], "name": "collapse_pl_onehot_lookup_select_compare_0_6", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "487e42631681a02a793e496ab694e8c775f2a57a6192ccf199db39c8884aa917", "shape": [ 6 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b89272f1a6c0413461abab168045ef49fbf9fb729dad42f0654f0dfd42739226", "shape": [ 6, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9abbdc8b936921abf739ee2c89ed6b5934597474c7daab7f2374a992ddd9abe1", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 6 }, "source_nodes": [ "s:1469" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=1.0005037927840583))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005037927840583, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005037927840583, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:987", "inputs": [ "l:988" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1473" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:988", "inputs": [ "l:989", "l:989", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "4a219c06af2467623290b375e580c952ecdae33edf14aa594ac393d27e76b67e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1474" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:989", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1475" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:990", "inputs": [ "l:991", "l:915", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "0ebfee506020ed8b5565a2f5252bea8bae295626330b537d4c9b488a46a712de", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "9010fceb49e61304a784d8dc3d60e4881a4749caa665c18a91caecfad9bbce83", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1477" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:991", "inputs": [ "l:987" ], "name": "collapse_pl_onehot_lookup_select_fused_add_const_multiply_const_0_13", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9e31d6a87d5297887430169a7c8916d576a79774aa52ce038c135be992e02beb", "shape": [ 13 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "50fb7e19db723be8683dcd3f4cf6e11a339ed542668aad586d6e1a4d8140363b", "shape": [ 13, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "751a6babe189f2ca9c0dad8222dd14f30df3cddad3e67ea11922607244ea0acb", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "63f3db08b598c24d44c4681319c24a91c666753171c27f9fd6040fa5c14b96d5", "shape": [ 13, 6 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 13 }, "source_nodes": [ "s:1478" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005018967050747233, hi=1.0005))" } ], "claimed_range": { "lower": -0.0005018967050747233, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005018967050747233, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:992", "inputs": [ "l:993" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1483" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0200249999999995))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:993", "inputs": [ "l:994", "l:998" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1484" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:994", "inputs": [ "l:995" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1485" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:995", "inputs": [ "l:996", "l:997" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1487" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:996", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "aa35d7d5d29f2463284c11f8021afed217eca0782d8286f903a06fa4b8e5fe00", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1488" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:997", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c6f408755f3e0820541405e08e18c2b65515c3a800b5113c379c2596aadbe210", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1489" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:998", "inputs": [ "l:999" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1491" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:999", "inputs": [ "l:1000", "l:1003", "l:1006" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1492" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:1000", "inputs": [ "l:1001" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1493" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1001", "inputs": [ "l:34", "l:1002" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1494" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:1002", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1495" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1003", "inputs": [ "l:1004" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1496" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1004", "inputs": [ "l:60", "l:1005" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1497" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:1005", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1498" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1006", "inputs": [ "l:1007" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1499" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1007", "inputs": [ "l:69", "l:1008" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1500" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:1008", "inputs": [ "l:1009" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:1501" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1009", "inputs": [ "l:1010", "l:909", "l:1014" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1502" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:1010", "inputs": [ "l:1011" ], "name": "collapse_pl_onehot_lookup_select_compare_0_5", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "781cfb29c45899268c552a509ab3a88e82464fecafd2e15065f93f1db878ba1b", "shape": [ 5 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5ad4da4820043d54431225b4e36be9a6e06537d936de5c79031482ba801119fc", "shape": [ 5, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4228d9323807eb3ac3cab0630e9c37fe351b38630bc943cd0338aa9c08ff873b", "shape": [ 5, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 5 }, "source_nodes": [ "s:1503" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=1.0005037977642302))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005037977642302, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005037977642302, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:1011", "inputs": [ "l:1012" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1507" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:1012", "inputs": [ "l:1013", "l:1013", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1508" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1013", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1509" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:1014", "inputs": [ "l:1015", "l:915", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "0ebfee506020ed8b5565a2f5252bea8bae295626330b537d4c9b488a46a712de", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "9010fceb49e61304a784d8dc3d60e4881a4749caa665c18a91caecfad9bbce83", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1511" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1015", "inputs": [ "l:1011" ], "name": "collapse_pl_onehot_lookup_select_fused_add_const_multiply_const_0_8", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "303897b9dc7339bc01f2475d11dbd8881a7ffc503d65e25bfeb86f82bb49800a", "shape": [ 8 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "c371c7855c2491a905c694166d423bb80ca732928149d712c314d56e86c6405d", "shape": [ 8, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "5b2342f4326491e0410698a0fb40c024fc8f633eac9891a756e5c40c871bd66f", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "66ff72df5612661c641bd31bb49d4bea3cdeea9b47c5d6f13680ab85a16e0d72", "shape": [ 8, 6 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 8 }, "source_nodes": [ "s:1512" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005, hi=1.0005))" } ], "claimed_range": { "lower": -0.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:1016", "inputs": [ "l:1017" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1517" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0200249999999995))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1017", "inputs": [ "l:1018", "l:1022" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1518" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:1018", "inputs": [ "l:1019" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1519" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:1019", "inputs": [ "l:1020", "l:1021" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1521" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:1020", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "dee95cfd95fe9c0151dba42d63cf451b06d57f8e26bec9cde7ae4ccc858772aa", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1522" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:1021", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c139a06e54e1e0f0339c3619fdcd664073a35456fe9886d9798195e0e0379efb", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1523" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:1022", "inputs": [ "l:1023" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1525" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1023", "inputs": [ "l:1024", "l:1027", "l:1030" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1526" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:1024", "inputs": [ "l:1025" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1527" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1025", "inputs": [ "l:34", "l:1026" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1528" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:1026", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1529" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1027", "inputs": [ "l:1028" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1530" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1028", "inputs": [ "l:60", "l:1029" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1531" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:1029", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1532" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1030", "inputs": [ "l:1031" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1533" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1031", "inputs": [ "l:69", "l:1032" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1534" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:1032", "inputs": [ "l:1033" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "source_nodes": [ "s:1535" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1033", "inputs": [ "l:1034", "l:909", "l:1038" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1536" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "l:1034", "inputs": [ "l:1035" ], "name": "collapse_pl_onehot_lookup_select_compare_0_5", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "211453acaed59271aa1fef09c5ecb6f1770e8ef8dd500843cd09bfb819a85aa6", "shape": [ 5 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "36585f50d4b1a4b6488e5617171feccbe07157e8101ce9d8c2986d0447d9eeb5", "shape": [ 5, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9b7a95ca2869f588a57dcd175c97c826b03d2bac1da56b20a9812176802dcd00", "shape": [ 5, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 5 }, "source_nodes": [ "s:1537" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=1.0005038008999358))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005038008999358, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005038008999358, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:1035", "inputs": [ "l:1036" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1541" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:1036", "inputs": [ "l:1037", "l:1037", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "42861c9444db42b4d1d4965d39b2485c070ce6152a23485353d3514c029140fc", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1542" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1037", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1543" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:1038", "inputs": [ "l:1039", "l:915", "l:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "0ebfee506020ed8b5565a2f5252bea8bae295626330b537d4c9b488a46a712de", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "9010fceb49e61304a784d8dc3d60e4881a4749caa665c18a91caecfad9bbce83", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:1545" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1039", "inputs": [ "l:1035" ], "name": "collapse_pl_onehot_lookup_select_fused_add_const_multiply_const_0_4", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5685700af92b304fff7df4d8447d1cb99cd2acee0af93e1257a0617bd5d9cc44", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "64b467e8fea7287c693e3af453917b4e869052b8a6bbd8f776e976e599f1bafa", "shape": [ 4, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "9c96abcf679f283862a28bc89f5d9fa1085c5b3f360389f00ac996ca84570fd0", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "b30365178bd5aac01484a211f9ca440f8a3c3f24c8d4a0f216143164d6ddb997", "shape": [ 4, 6 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 4 }, "source_nodes": [ "s:1546" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005, hi=1.0005))" } ], "claimed_range": { "lower": -0.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:1040", "inputs": [ "l:1041" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1551" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1041", "inputs": [ "l:1042", "l:1046" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1552" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:1042", "inputs": [ "l:1043" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1553" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:1043", "inputs": [ "l:1044", "l:1045" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1555" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:1044", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "bbc16069d33e5cd6fc8fbe89aba5cbcd6379e2926aba86ed299b10c60134689d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1556" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:1045", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "7a1aad3244914967fc4e6a98bfa0537fa58ab41acab95f80be84f5e2a20bf0ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:1557" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:1046", "inputs": [ "l:1047" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1559" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1047", "inputs": [ "l:1048", "l:1051", "l:1054" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1560" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "l:1048", "inputs": [ "l:1049" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1561" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1049", "inputs": [ "l:34", "l:1050" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1562" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:1050", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1563" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1051", "inputs": [ "l:1052" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1564" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1052", "inputs": [ "l:60", "l:1053" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1565" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:1053", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1566" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1054", "inputs": [ "l:1055" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "source_nodes": [ "s:1567" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1055", "inputs": [ "l:69", "l:1056" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1568" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:1056", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1569" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "l:1057", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "219463451d29bbc5e3938d0f5394ec9a438fa2930a11e18f82abcb8091cc15f2", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:1570" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 } ], "root": "l:0", "sha256": "650161b68c52e30c52701195eeef3caae06d9da3f3078d00d5ae5d31348c37ed" }, "realization_map": [ { "lowered": "l:0", "source": "s:0", "status": "whole" }, { "lowered": "l:1", "source": "s:1", "status": "whole" }, { "lowered": "l:2", "source": "s:2", "status": "whole" }, { "lowered": "l:3", "source": "s:3", "status": "whole" }, { "lowered": "l:4", "source": "s:4", "status": "whole" }, { "lowered": "l:5", "source": "s:5", "status": "whole" }, { "lowered": "l:6", "source": "s:6", "status": "whole" }, { "lowered": "l:7", "source": "s:7", "status": "whole" }, { "lowered": "l:8", "source": "s:8", "status": "whole" }, { "lowered": "l:9", "source": "s:9", "status": "whole" }, { "lowered": "l:10", "source": "s:10", "status": "whole" }, { "lowered": "l:11", "source": "s:11", "status": "whole" }, { "lowered": "l:12", "source": "s:12", "status": "whole" }, { "lowered": "l:13", "source": "s:13", "status": "whole" }, { "lowered": "l:14", "source": "s:14", "status": "whole" }, { "source": "s:15", "status": "not_materialized" }, { "lowered": "l:15", "source": "s:16", "status": "whole" }, { "lowered": "l:16", "source": "s:17", "status": "whole" }, { "lowered": "l:17", "source": "s:18", "status": "whole" }, { "lowered": "l:18", "source": "s:19", "status": "whole" }, { "lowered": "l:19", "source": "s:20", "status": "whole" }, { "lowered": "l:20", "source": "s:21", "status": "whole" }, { "lowered": "l:21", "source": "s:22", "status": "whole" }, { "lowered": "l:22", "source": "s:23", "status": "whole" }, { "lowered": "l:23", "source": "s:24", "status": "whole" }, { "lowered": "l:24", "source": "s:25", "status": "whole" }, { "lowered": "l:25", "source": "s:26", "status": "whole" }, { "lowered": "l:26", "source": "s:27", "status": "whole" }, { "lowered": "l:27", "source": "s:28", "status": "whole" }, { "lowered": "l:28", "source": "s:29", "status": "whole" }, { "source": "s:30", "status": "not_materialized" }, { "lowered": "l:29", "source": "s:31", "status": "whole" }, { "source": "s:32", "status": "not_materialized" }, { "lowered": "l:30", "source": "s:33", "status": "whole" }, { "lowered": "l:31", "source": "s:34", "status": "whole" }, { "lowered": "l:32", "source": "s:35", "status": "whole" }, { "lowered": "l:33", "source": "s:36", "status": "whole" }, { "lowered": "l:34", "source": "s:37", "status": "whole" }, { "lowered": "l:35", "source": "s:38", "status": "whole" }, { "lowered": "l:36", "source": "s:39", "status": "whole" }, { "lowered": "l:37", "source": "s:40", "status": "whole" }, { "lowered": "l:38", "source": "s:41", "status": "whole" }, { "lowered": "l:39", "source": "s:42", "status": "whole" }, { "source": "s:43", "status": "not_materialized" }, { "lowered": "l:40", "source": "s:44", "status": "whole" }, { "lowered": "l:41", "source": "s:45", "status": "whole" }, { "source": "s:46", "status": "not_materialized" }, { "lowered": "l:42", "source": "s:47", "status": "whole" }, { "lowered": "l:43", "source": "s:48", "status": "whole" }, { "lowered": "l:44", "source": "s:49", "status": "whole" }, { "lowered": "l:45", "source": "s:50", "status": "whole" }, { "lowered": "l:46", "source": "s:51", "status": "whole" }, { "lowered": "l:47", "source": "s:52", "status": "whole" }, { "lowered": "l:48", "source": "s:53", "status": "whole" }, { "lowered": "l:49", "source": "s:54", "status": "whole" }, { "lowered": "l:50", "source": "s:55", "status": "whole" }, { "lowered": "l:51", "source": "s:56", "status": "whole" }, { "lowered": "l:52", "source": "s:57", "status": "whole" }, { "lowered": "l:53", "source": "s:58", "status": "whole" }, { "lowered": "l:54", "source": "s:59", "status": "whole" }, { "lowered": "l:55", "source": "s:60", "status": "whole" }, { "lowered": "l:56", "source": "s:61", "status": "whole" }, { "lowered": "l:57", "source": "s:62", "status": "whole" }, { "lowered": "l:58", "source": "s:63", "status": "whole" }, { "lowered": "l:59", "source": "s:64", "status": "whole" }, { "lowered": "l:60", "source": "s:65", "status": "whole" }, { "lowered": "l:61", "source": "s:66", "status": "whole" }, { "lowered": "l:62", "source": "s:67", "status": "whole" }, { "lowered": "l:63", "source": "s:68", "status": "whole" }, { "lowered": "l:64", "source": "s:69", "status": "whole" }, { "lowered": "l:65", "source": "s:70", "status": "whole" }, { "lowered": "l:66", "source": "s:71", "status": "whole" }, { "lowered": "l:67", "source": "s:72", "status": "whole" }, { "lowered": "l:68", "source": "s:73", "status": "whole" }, { "lowered": "l:69", "source": "s:74", "status": "whole" }, { "lowered": "l:70", "source": "s:75", "status": "whole" }, { "lowered": "l:71", "source": "s:76", "status": "whole" }, { "lowered": "l:72", "source": "s:77", "status": "whole" }, { "lowered": "l:73", "source": "s:78", "status": "whole" }, { "lowered": "l:74", "source": "s:79", "status": "whole" }, { "lowered": "l:75", "source": "s:80", "status": "whole" }, { "lowered": "l:76", "source": "s:81", "status": "whole" }, { "lowered": "l:77", "source": "s:82", "status": "whole" }, { "lowered": "l:78", "source": "s:83", "status": "whole" }, { "source": "s:84", "status": "not_materialized" }, { "lowered": "l:79", "source": "s:85", "status": "whole" }, { "lowered": "l:80", "source": "s:86", "status": "whole" }, { "lowered": "l:81", "source": "s:87", "status": "whole" }, { "source": "s:88", "status": "not_materialized" }, { "lowered": "l:82", "source": "s:89", "status": "whole" }, { "lowered": "l:83", "source": "s:90", "status": "whole" }, { "lowered": "l:84", "source": "s:91", "status": "whole" }, { "lowered": "l:85", "source": "s:92", "status": "whole" }, { "lowered": "l:86", "source": "s:93", "status": "whole" }, { "source": "s:94", "status": "not_materialized" }, { "source": "s:95", "status": "not_materialized" }, { "source": "s:96", "status": "not_materialized" }, { "source": "s:97", "status": "not_materialized" }, { "source": "s:98", "status": "not_materialized" }, { "lowered": "l:87", "source": "s:99", "status": "whole" }, { "lowered": "l:88", "source": "s:100", "status": "whole" }, { "lowered": "l:89", "source": "s:101", "status": "whole" }, { "lowered": "l:90", "source": "s:102", "status": "whole" }, { "lowered": "l:91", "source": "s:103", "status": "whole" }, { "lowered": "l:92", "source": "s:104", "status": "whole" }, { "lowered": "l:93", "source": "s:105", "status": "whole" }, { "lowered": "l:94", "source": "s:106", "status": "whole" }, { "lowered": "l:95", "source": "s:107", "status": "whole" }, { "lowered": "l:96", "source": "s:108", "status": "whole" }, { "source": "s:109", "status": "not_materialized" }, { "source": "s:110", "status": "not_materialized" }, { "source": "s:111", "status": "not_materialized" }, { "source": "s:112", "status": "not_materialized" }, { "source": "s:113", "status": "not_materialized" }, { "source": "s:114", "status": "not_materialized" }, { "lowered": "l:97", "source": "s:115", "status": "whole" }, { "lowered": "l:98", "source": "s:116", "status": "whole" }, { "lowered": "l:99", "source": "s:117", "status": "whole" }, { "lowered": "l:100", "source": "s:118", "status": "whole" }, { "lowered": "l:101", "source": "s:119", "status": "whole" }, { "lowered": "l:102", "source": "s:120", "status": "whole" }, { "lowered": "l:103", "source": "s:121", "status": "whole" }, { "lowered": "l:104", "source": "s:122", "status": "whole" }, { "lowered": "l:105", "source": "s:123", "status": "whole" }, { "lowered": "l:106", "source": "s:124", "status": "whole" }, { "lowered": "l:107", "source": "s:125", "status": "whole" }, { "lowered": "l:108", "source": "s:126", "status": "whole" }, { "lowered": "l:109", "source": "s:127", "status": "whole" }, { "lowered": "l:110", "source": "s:128", "status": "whole" }, { "lowered": "l:111", "source": "s:129", "status": "whole" }, { "lowered": "l:112", "source": "s:130", "status": "whole" }, { "lowered": "l:113", "source": "s:131", "status": "whole" }, { "lowered": "l:114", "source": "s:132", "status": "whole" }, { "source": "s:133", "status": "not_materialized" }, { "source": "s:134", "status": "not_materialized" }, { "source": "s:135", "status": "not_materialized" }, { "source": "s:136", "status": "not_materialized" }, { "lowered": "l:115", "source": "s:137", "status": "whole" }, { "lowered": "l:116", "source": "s:138", "status": "whole" }, { "lowered": "l:117", "source": "s:139", "status": "whole" }, { "lowered": "l:118", "source": "s:140", "status": "whole" }, { "lowered": "l:119", "source": "s:141", "status": "whole" }, { "source": "s:142", "status": "not_materialized" }, { "lowered": "l:120", "source": "s:143", "status": "whole" }, { "lowered": "l:121", "source": "s:144", "status": "whole" }, { "lowered": "l:122", "source": "s:145", "status": "whole" }, { "lowered": "l:123", "source": "s:146", "status": "whole" }, { "lowered": "l:124", "source": "s:147", "status": "whole" }, { "lowered": "l:125", "source": "s:148", "status": "whole" }, { "lowered": "l:126", "source": "s:149", "status": "whole" }, { "lowered": "l:127", "source": "s:150", "status": "whole" }, { "lowered": "l:128", "source": "s:151", "status": "whole" }, { "lowered": "l:129", "source": "s:152", "status": "whole" }, { "source": "s:153", "status": "not_materialized" }, { "lowered": "l:130", "source": "s:154", "status": "whole" }, { "lowered": "l:131", "source": "s:155", "status": "whole" }, { "lowered": "l:132", "source": "s:156", "status": "whole" }, { "lowered": "l:133", "source": "s:157", "status": "whole" }, { "lowered": "l:134", "source": "s:158", "status": "whole" }, { "lowered": "l:135", "source": "s:159", "status": "whole" }, { "lowered": "l:136", "source": "s:160", "status": "whole" }, { "lowered": "l:137", "source": "s:161", "status": "whole" }, { "lowered": "l:138", "source": "s:162", "status": "whole" }, { "lowered": "l:139", "source": "s:163", "status": "whole" }, { "lowered": "l:140", "source": "s:164", "status": "whole" }, { "lowered": "l:141", "source": "s:165", "status": "whole" }, { "lowered": "l:142", "source": "s:166", "status": "whole" }, { "lowered": "l:143", "source": "s:167", "status": "whole" }, { "lowered": "l:144", "source": "s:168", "status": "whole" }, { "lowered": "l:145", "source": "s:169", "status": "whole" }, { "lowered": "l:146", "source": "s:170", "status": "whole" }, { "lowered": "l:147", "source": "s:171", "status": "whole" }, { "lowered": "l:148", "source": "s:172", "status": "whole" }, { "lowered": "l:149", "source": "s:173", "status": "whole" }, { "source": "s:174", "status": "not_materialized" }, { "source": "s:175", "status": "not_materialized" }, { "source": "s:176", "status": "not_materialized" }, { "source": "s:177", "status": "not_materialized" }, { "source": "s:178", "status": "not_materialized" }, { "source": "s:179", "status": "not_materialized" }, { "lowered": "l:150", "source": "s:180", "status": "whole" }, { "lowered": "l:151", "source": "s:181", "status": "whole" }, { "lowered": "l:152", "source": "s:182", "status": "whole" }, { "lowered": "l:153", "source": "s:183", "status": "whole" }, { "lowered": "l:154", "source": "s:184", "status": "whole" }, { "lowered": "l:155", "source": "s:185", "status": "whole" }, { "lowered": "l:156", "source": "s:186", "status": "whole" }, { "lowered": "l:157", "source": "s:187", "status": "whole" }, { "lowered": "l:158", "source": "s:188", "status": "whole" }, { "lowered": "l:159", "source": "s:189", "status": "whole" }, { "lowered": "l:160", "source": "s:190", "status": "whole" }, { "lowered": "l:161", "source": "s:191", "status": "whole" }, { "lowered": "l:162", "source": "s:192", "status": "whole" }, { "lowered": "l:163", "source": "s:193", "status": "whole" }, { "lowered": "l:164", "source": "s:194", "status": "whole" }, { "lowered": "l:165", "source": "s:195", "status": "whole" }, { "lowered": "l:166", "source": "s:196", "status": "whole" }, { "source": "s:197", "status": "not_materialized" }, { "source": "s:198", "status": "not_materialized" }, { "source": "s:199", "status": "not_materialized" }, { "source": "s:200", "status": "not_materialized" }, { "lowered": "l:167", "source": "s:201", "status": "whole" }, { "lowered": "l:168", "source": "s:202", "status": "whole" }, { "lowered": "l:169", "source": "s:203", "status": "whole" }, { "lowered": "l:170", "source": "s:204", "status": "whole" }, { "lowered": "l:171", "source": "s:205", "status": "whole" }, { "source": "s:206", "status": "not_materialized" }, { "lowered": "l:172", "source": "s:207", "status": "whole" }, { "lowered": "l:173", "source": "s:208", "status": "whole" }, { "lowered": "l:174", "source": "s:209", "status": "whole" }, { "source": "s:210", "status": "not_materialized" }, { "lowered": "l:175", "source": "s:211", "status": "whole" }, { "lowered": "l:176", "source": "s:212", "status": "whole" }, { "lowered": "l:177", "source": "s:213", "status": "whole" }, { "lowered": "l:178", "source": "s:214", "status": "whole" }, { "lowered": "l:179", "source": "s:215", "status": "whole" }, { "lowered": "l:180", "source": "s:216", "status": "whole" }, { "lowered": "l:181", "source": "s:217", "status": "whole" }, { "lowered": "l:182", "source": "s:218", "status": "whole" }, { "lowered": "l:183", "source": "s:219", "status": "whole" }, { "lowered": "l:184", "source": "s:220", "status": "whole" }, { "lowered": "l:185", "source": "s:221", "status": "whole" }, { "source": "s:222", "status": "not_materialized" }, { "source": "s:223", "status": "not_materialized" }, { "lowered": "l:186", "source": "s:224", "status": "whole" }, { "lowered": "l:187", "source": "s:225", "status": "whole" }, { "lowered": "l:188", "source": "s:226", "status": "whole" }, { "lowered": "l:189", "source": "s:227", "status": "whole" }, { "lowered": "l:190", "source": "s:228", "status": "whole" }, { "lowered": "l:191", "source": "s:229", "status": "whole" }, { "lowered": "l:192", "source": "s:230", "status": "whole" }, { "lowered": "l:193", "source": "s:231", "status": "whole" }, { "lowered": "l:194", "source": "s:232", "status": "whole" }, { "lowered": "l:195", "source": "s:233", "status": "whole" }, { "lowered": "l:196", "source": "s:234", "status": "whole" }, { "lowered": "l:197", "source": "s:235", "status": "whole" }, { "lowered": "l:198", "source": "s:236", "status": "whole" }, { "lowered": "l:199", "source": "s:237", "status": "whole" }, { "source": "s:238", "status": "not_materialized" }, { "source": "s:239", "status": "not_materialized" }, { "source": "s:240", "status": "not_materialized" }, { "source": "s:241", "status": "not_materialized" }, { "source": "s:242", "status": "not_materialized" }, { "source": "s:243", "status": "not_materialized" }, { "source": "s:244", "status": "not_materialized" }, { "source": "s:245", "status": "not_materialized" }, { "lowered": "l:200", "source": "s:246", "status": "whole" }, { "lowered": "l:201", "source": "s:247", "status": "whole" }, { "lowered": "l:202", "source": "s:248", "status": "whole" }, { "lowered": "l:203", "source": "s:249", "status": "whole" }, { "lowered": "l:204", "source": "s:250", "status": "whole" }, { "lowered": "l:205", "source": "s:251", "status": "whole" }, { "lowered": "l:206", "source": "s:252", "status": "whole" }, { "lowered": "l:207", "source": "s:253", "status": "whole" }, { "source": "s:254", "status": "not_materialized" }, { "source": "s:255", "status": "not_materialized" }, { "source": "s:256", "status": "not_materialized" }, { "source": "s:257", "status": "not_materialized" }, { "source": "s:258", "status": "not_materialized" }, { "source": "s:259", "status": "not_materialized" }, { "source": "s:260", "status": "not_materialized" }, { "source": "s:261", "status": "not_materialized" }, { "lowered": "l:208", "source": "s:262", "status": "whole" }, { "lowered": "l:209", "source": "s:263", "status": "whole" }, { "lowered": "l:210", "source": "s:264", "status": "whole" }, { "source": "s:265", "status": "not_materialized" }, { "source": "s:266", "status": "not_materialized" }, { "source": "s:267", "status": "not_materialized" }, { "source": "s:268", "status": "not_materialized" }, { "source": "s:269", "status": "not_materialized" }, { "lowered": "l:621", "source": "s:270", "status": "whole" }, { "source": "s:271", "status": "not_materialized" }, { "source": "s:272", "status": "not_materialized" }, { "lowered": "l:258", "source": "s:273", "status": "whole" }, { "lowered": "l:259", "source": "s:274", "status": "whole" }, { "source": "s:275", "status": "not_materialized" }, { "source": "s:276", "status": "not_materialized" }, { "lowered": "l:211", "source": "s:277", "status": "whole" }, { "lowered": "l:212", "source": "s:278", "status": "whole" }, { "lowered": "l:213", "source": "s:279", "status": "whole" }, { "source": "s:280", "status": "not_materialized" }, { "lowered": "l:214", "source": "s:281", "status": "whole" }, { "lowered": "l:215", "source": "s:282", "status": "whole" }, { "lowered": "l:216", "source": "s:283", "status": "whole" }, { "source": "s:284", "status": "not_materialized" }, { "lowered": "l:217", "source": "s:285", "status": "whole" }, { "lowered": "l:218", "source": "s:286", "status": "whole" }, { "lowered": "l:219", "source": "s:287", "status": "whole" }, { "lowered": "l:220", "source": "s:288", "status": "whole" }, { "lowered": "l:221", "source": "s:289", "status": "whole" }, { "source": "s:290", "status": "not_materialized" }, { "source": "s:291", "status": "not_materialized" }, { "source": "s:292", "status": "not_materialized" }, { "source": "s:293", "status": "not_materialized" }, { "lowered": "l:222", "source": "s:294", "status": "whole" }, { "lowered": "l:223", "source": "s:295", "status": "whole" }, { "lowered": "l:224", "source": "s:296", "status": "whole" }, { "lowered": "l:225", "source": "s:297", "status": "whole" }, { "lowered": "l:226", "source": "s:298", "status": "whole" }, { "lowered": "l:227", "source": "s:299", "status": "whole" }, { "lowered": "l:228", "source": "s:300", "status": "whole" }, { "lowered": "l:229", "source": "s:301", "status": "whole" }, { "lowered": "l:230", "source": "s:302", "status": "whole" }, { "lowered": "l:231", "source": "s:303", "status": "whole" }, { "lowered": "l:232", "source": "s:304", "status": "whole" }, { "source": "s:305", "status": "not_materialized" }, { "source": "s:306", "status": "not_materialized" }, { "source": "s:307", "status": "not_materialized" }, { "source": "s:308", "status": "not_materialized" }, { "source": "s:309", "status": "not_materialized" }, { "source": "s:310", "status": "not_materialized" }, { "source": "s:311", "status": "not_materialized" }, { "source": "s:312", "status": "not_materialized" }, { "lowered": "l:233", "source": "s:313", "status": "whole" }, { "lowered": "l:234", "source": "s:314", "status": "whole" }, { "lowered": "l:235", "source": "s:315", "status": "whole" }, { "lowered": "l:236", "source": "s:316", "status": "whole" }, { "lowered": "l:237", "source": "s:317", "status": "whole" }, { "lowered": "l:238", "source": "s:318", "status": "whole" }, { "lowered": "l:239", "source": "s:319", "status": "whole" }, { "lowered": "l:240", "source": "s:320", "status": "whole" }, { "lowered": "l:241", "source": "s:321", "status": "whole" }, { "source": "s:322", "status": "not_materialized" }, { "source": "s:323", "status": "not_materialized" }, { "source": "s:324", "status": "not_materialized" }, { "source": "s:325", "status": "not_materialized" }, { "source": "s:326", "status": "not_materialized" }, { "source": "s:327", "status": "not_materialized" }, { "source": "s:328", "status": "not_materialized" }, { "source": "s:329", "status": "not_materialized" }, { "lowered": "l:242", "source": "s:330", "status": "whole" }, { "source": "s:331", "status": "not_materialized" }, { "lowered": "l:243", "source": "s:332", "status": "whole" }, { "lowered": "l:244", "source": "s:333", "status": "whole" }, { "lowered": "l:245", "source": "s:334", "status": "whole" }, { "lowered": "l:246", "source": "s:335", "status": "whole" }, { "lowered": "l:247", "source": "s:336", "status": "whole" }, { "lowered": "l:248", "source": "s:337", "status": "whole" }, { "lowered": "l:249", "source": "s:338", "status": "whole" }, { "lowered": "l:250", "source": "s:339", "status": "whole" }, { "source": "s:340", "status": "not_materialized" }, { "source": "s:341", "status": "not_materialized" }, { "source": "s:342", "status": "not_materialized" }, { "source": "s:343", "status": "not_materialized" }, { "lowered": "l:251", "source": "s:344", "status": "whole" }, { "lowered": "l:252", "source": "s:345", "status": "whole" }, { "source": "s:346", "status": "not_materialized" }, { "source": "s:347", "status": "not_materialized" }, { "lowered": "l:254", "source": "s:348", "status": "whole" }, { "lowered": "l:255", "source": "s:349", "status": "whole" }, { "source": "s:350", "status": "not_materialized" }, { "lowered": "l:256", "source": "s:351", "status": "whole" }, { "lowered": "l:257", "source": "s:352", "status": "whole" }, { "source": "s:353", "status": "not_materialized" }, { "lowered": "l:260", "source": "s:354", "status": "whole" }, { "source": "s:355", "status": "not_materialized" }, { "lowered": "l:261", "source": "s:356", "status": "whole" }, { "lowered": "l:262", "source": "s:357", "status": "whole" }, { "lowered": "l:263", "source": "s:358", "status": "whole" }, { "source": "s:359", "status": "not_materialized" }, { "lowered": "l:264", "source": "s:360", "status": "whole" }, { "lowered": "l:265", "source": "s:361", "status": "whole" }, { "lowered": "l:266", "source": "s:362", "status": "whole" }, { "source": "s:363", "status": "not_materialized" }, { "lowered": "l:267", "source": "s:364", "status": "whole" }, { "lowered": "l:268", "source": "s:365", "status": "whole" }, { "lowered": "l:269", "source": "s:366", "status": "whole" }, { "lowered": "l:270", "source": "s:367", "status": "whole" }, { "lowered": "l:271", "source": "s:368", "status": "whole" }, { "source": "s:369", "status": "not_materialized" }, { "source": "s:370", "status": "not_materialized" }, { "source": "s:371", "status": "not_materialized" }, { "source": "s:372", "status": "not_materialized" }, { "lowered": "l:272", "source": "s:373", "status": "whole" }, { "lowered": "l:273", "source": "s:374", "status": "whole" }, { "lowered": "l:274", "source": "s:375", "status": "whole" }, { "lowered": "l:275", "source": "s:376", "status": "whole" }, { "source": "s:377", "status": "not_materialized" }, { "lowered": "l:276", "source": "s:378", "status": "whole" }, { "lowered": "l:277", "source": "s:379", "status": "whole" }, { "lowered": "l:278", "source": "s:380", "status": "whole" }, { "lowered": "l:279", "source": "s:381", "status": "whole" }, { "lowered": "l:280", "source": "s:382", "status": "whole" }, { "lowered": "l:281", "source": "s:383", "status": "whole" }, { "lowered": "l:282", "source": "s:384", "status": "whole" }, { "lowered": "l:283", "source": "s:385", "status": "whole" }, { "source": "s:386", "status": "not_materialized" }, { "source": "s:387", "status": "not_materialized" }, { "source": "s:388", "status": "not_materialized" }, { "source": "s:389", "status": "not_materialized" }, { "lowered": "l:284", "source": "s:390", "status": "whole" }, { "lowered": "l:285", "source": "s:391", "status": "whole" }, { "source": "s:392", "status": "not_materialized" }, { "source": "s:393", "status": "not_materialized" }, { "lowered": "l:287", "source": "s:394", "status": "whole" }, { "lowered": "l:288", "source": "s:395", "status": "whole" }, { "source": "s:396", "status": "not_materialized" }, { "lowered": "l:289", "source": "s:397", "status": "whole" }, { "lowered": "l:290", "source": "s:398", "status": "whole" }, { "source": "s:399", "status": "not_materialized" }, { "source": "s:400", "status": "not_materialized" }, { "lowered": "l:291", "source": "s:401", "status": "whole" }, { "lowered": "l:292", "source": "s:402", "status": "whole" }, { "source": "s:403", "status": "not_materialized" }, { "source": "s:404", "status": "not_materialized" }, { "lowered": "l:293", "source": "s:405", "status": "whole" }, { "lowered": "l:294", "source": "s:406", "status": "whole" }, { "lowered": "l:295", "source": "s:407", "status": "whole" }, { "source": "s:408", "status": "not_materialized" }, { "lowered": "l:296", "source": "s:409", "status": "whole" }, { "lowered": "l:297", "source": "s:410", "status": "whole" }, { "lowered": "l:298", "source": "s:411", "status": "whole" }, { "source": "s:412", "status": "not_materialized" }, { "lowered": "l:299", "source": "s:413", "status": "whole" }, { "lowered": "l:300", "source": "s:414", "status": "whole" }, { "lowered": "l:301", "source": "s:415", "status": "whole" }, { "lowered": "l:302", "source": "s:416", "status": "whole" }, { "lowered": "l:303", "source": "s:417", "status": "whole" }, { "source": "s:418", "status": "not_materialized" }, { "source": "s:419", "status": "not_materialized" }, { "source": "s:420", "status": "not_materialized" }, { "source": "s:421", "status": "not_materialized" }, { "source": "s:422", "status": "not_materialized" }, { "source": "s:423", "status": "not_materialized" }, { "source": "s:424", "status": "not_materialized" }, { "lowered": "l:304", "source": "s:425", "status": "whole" }, { "lowered": "l:305", "source": "s:426", "status": "whole" }, { "lowered": "l:306", "source": "s:427", "status": "whole" }, { "source": "s:428", "status": "not_materialized" }, { "source": "s:429", "status": "not_materialized" }, { "source": "s:430", "status": "not_materialized" }, { "source": "s:431", "status": "not_materialized" }, { "source": "s:432", "status": "not_materialized" }, { "source": "s:433", "status": "not_materialized" }, { "lowered": "l:307", "source": "s:434", "status": "whole" }, { "lowered": "l:308", "source": "s:435", "status": "whole" }, { "lowered": "l:309", "source": "s:436", "status": "whole" }, { "lowered": "l:310", "source": "s:437", "status": "whole" }, { "lowered": "l:311", "source": "s:438", "status": "whole" }, { "lowered": "l:312", "source": "s:439", "status": "whole" }, { "lowered": "l:313", "source": "s:440", "status": "whole" }, { "lowered": "l:314", "source": "s:441", "status": "whole" }, { "lowered": "l:315", "source": "s:442", "status": "whole" }, { "lowered": "l:316", "source": "s:443", "status": "whole" }, { "source": "s:444", "status": "not_materialized" }, { "source": "s:445", "status": "not_materialized" }, { "source": "s:446", "status": "not_materialized" }, { "lowered": "l:317", "source": "s:447", "status": "whole" }, { "lowered": "l:318", "source": "s:448", "status": "whole" }, { "lowered": "l:319", "source": "s:449", "status": "whole" }, { "source": "s:450", "status": "not_materialized" }, { "source": "s:451", "status": "not_materialized" }, { "source": "s:452", "status": "not_materialized" }, { "source": "s:453", "status": "not_materialized" }, { "lowered": "l:320", "source": "s:454", "status": "whole" }, { "lowered": "l:321", "source": "s:455", "status": "whole" }, { "source": "s:456", "status": "not_materialized" }, { "source": "s:457", "status": "not_materialized" }, { "lowered": "l:323", "source": "s:458", "status": "whole" }, { "lowered": "l:324", "source": "s:459", "status": "whole" }, { "source": "s:460", "status": "not_materialized" }, { "source": "s:461", "status": "not_materialized" }, { "lowered": "l:325", "source": "s:462", "status": "whole" }, { "lowered": "l:326", "source": "s:463", "status": "whole" }, { "source": "s:464", "status": "not_materialized" }, { "source": "s:465", "status": "not_materialized" }, { "source": "s:466", "status": "not_materialized" }, { "lowered": "l:327", "source": "s:467", "status": "whole" }, { "lowered": "l:328", "source": "s:468", "status": "whole" }, { "lowered": "l:329", "source": "s:469", "status": "whole" }, { "source": "s:470", "status": "not_materialized" }, { "lowered": "l:330", "source": "s:471", "status": "whole" }, { "lowered": "l:331", "source": "s:472", "status": "whole" }, { "lowered": "l:332", "source": "s:473", "status": "whole" }, { "source": "s:474", "status": "not_materialized" }, { "lowered": "l:333", "source": "s:475", "status": "whole" }, { "lowered": "l:334", "source": "s:476", "status": "whole" }, { "lowered": "l:335", "source": "s:477", "status": "whole" }, { "lowered": "l:336", "source": "s:478", "status": "whole" }, { "lowered": "l:337", "source": "s:479", "status": "whole" }, { "source": "s:480", "status": "not_materialized" }, { "source": "s:481", "status": "not_materialized" }, { "source": "s:482", "status": "not_materialized" }, { "source": "s:483", "status": "not_materialized" }, { "source": "s:484", "status": "not_materialized" }, { "lowered": "l:338", "source": "s:485", "status": "whole" }, { "lowered": "l:339", "source": "s:486", "status": "whole" }, { "lowered": "l:340", "source": "s:487", "status": "whole" }, { "source": "s:488", "status": "not_materialized" }, { "lowered": "l:341", "source": "s:489", "status": "whole" }, { "lowered": "l:342", "source": "s:490", "status": "whole" }, { "lowered": "l:343", "source": "s:491", "status": "whole" }, { "source": "s:492", "status": "not_materialized" }, { "source": "s:493", "status": "not_materialized" }, { "source": "s:494", "status": "not_materialized" }, { "source": "s:495", "status": "not_materialized" }, { "source": "s:496", "status": "not_materialized" }, { "lowered": "l:344", "source": "s:497", "status": "whole" }, { "lowered": "l:345", "source": "s:498", "status": "whole" }, { "lowered": "l:346", "source": "s:499", "status": "whole" }, { "lowered": "l:347", "source": "s:500", "status": "whole" }, { "lowered": "l:348", "source": "s:501", "status": "whole" }, { "lowered": "l:349", "source": "s:502", "status": "whole" }, { "lowered": "l:350", "source": "s:503", "status": "whole" }, { "lowered": "l:351", "source": "s:504", "status": "whole" }, { "lowered": "l:352", "source": "s:505", "status": "whole" }, { "lowered": "l:353", "source": "s:506", "status": "whole" }, { "lowered": "l:354", "source": "s:507", "status": "whole" }, { "lowered": "l:355", "source": "s:508", "status": "whole" }, { "lowered": "l:356", "source": "s:509", "status": "whole" }, { "lowered": "l:357", "source": "s:510", "status": "whole" }, { "lowered": "l:358", "source": "s:511", "status": "whole" }, { "source": "s:512", "status": "not_materialized" }, { "lowered": "l:359", "source": "s:513", "status": "whole" }, { "lowered": "l:360", "source": "s:514", "status": "whole" }, { "lowered": "l:361", "source": "s:515", "status": "whole" }, { "source": "s:516", "status": "not_materialized" }, { "source": "s:517", "status": "not_materialized" }, { "source": "s:518", "status": "not_materialized" }, { "source": "s:519", "status": "not_materialized" }, { "lowered": "l:362", "source": "s:520", "status": "whole" }, { "lowered": "l:363", "source": "s:521", "status": "whole" }, { "source": "s:522", "status": "not_materialized" }, { "source": "s:523", "status": "not_materialized" }, { "lowered": "l:365", "source": "s:524", "status": "whole" }, { "lowered": "l:366", "source": "s:525", "status": "whole" }, { "source": "s:526", "status": "not_materialized" }, { "source": "s:527", "status": "not_materialized" }, { "source": "s:528", "status": "not_materialized" }, { "lowered": "l:367", "source": "s:529", "status": "whole" }, { "lowered": "l:368", "source": "s:530", "status": "whole" }, { "lowered": "l:369", "source": "s:531", "status": "whole" }, { "source": "s:532", "status": "not_materialized" }, { "lowered": "l:370", "source": "s:533", "status": "whole" }, { "lowered": "l:371", "source": "s:534", "status": "whole" }, { "lowered": "l:372", "source": "s:535", "status": "whole" }, { "source": "s:536", "status": "not_materialized" }, { "lowered": "l:373", "source": "s:537", "status": "whole" }, { "lowered": "l:374", "source": "s:538", "status": "whole" }, { "lowered": "l:375", "source": "s:539", "status": "whole" }, { "lowered": "l:376", "source": "s:540", "status": "whole" }, { "lowered": "l:377", "source": "s:541", "status": "whole" }, { "source": "s:542", "status": "not_materialized" }, { "source": "s:543", "status": "not_materialized" }, { "source": "s:544", "status": "not_materialized" }, { "source": "s:545", "status": "not_materialized" }, { "lowered": "l:378", "source": "s:546", "status": "whole" }, { "lowered": "l:379", "source": "s:547", "status": "whole" }, { "lowered": "l:380", "source": "s:548", "status": "whole" }, { "lowered": "l:381", "source": "s:549", "status": "whole" }, { "source": "s:550", "status": "not_materialized" }, { "lowered": "l:382", "source": "s:551", "status": "whole" }, { "lowered": "l:383", "source": "s:552", "status": "whole" }, { "lowered": "l:384", "source": "s:553", "status": "whole" }, { "source": "s:554", "status": "not_materialized" }, { "source": "s:555", "status": "not_materialized" }, { "source": "s:556", "status": "not_materialized" }, { "source": "s:557", "status": "not_materialized" }, { "source": "s:558", "status": "not_materialized" }, { "lowered": "l:385", "source": "s:559", "status": "whole" }, { "lowered": "l:386", "source": "s:560", "status": "whole" }, { "lowered": "l:387", "source": "s:561", "status": "whole" }, { "lowered": "l:388", "source": "s:562", "status": "whole" }, { "lowered": "l:389", "source": "s:563", "status": "whole" }, { "lowered": "l:390", "source": "s:564", "status": "whole" }, { "lowered": "l:391", "source": "s:565", "status": "whole" }, { "lowered": "l:392", "source": "s:566", "status": "whole" }, { "lowered": "l:393", "source": "s:567", "status": "whole" }, { "lowered": "l:394", "source": "s:568", "status": "whole" }, { "lowered": "l:395", "source": "s:569", "status": "whole" }, { "lowered": "l:396", "source": "s:570", "status": "whole" }, { "lowered": "l:397", "source": "s:571", "status": "whole" }, { "lowered": "l:398", "source": "s:572", "status": "whole" }, { "source": "s:573", "status": "not_materialized" }, { "lowered": "l:399", "source": "s:574", "status": "whole" }, { "lowered": "l:400", "source": "s:575", "status": "whole" }, { "lowered": "l:401", "source": "s:576", "status": "whole" }, { "source": "s:577", "status": "not_materialized" }, { "source": "s:578", "status": "not_materialized" }, { "source": "s:579", "status": "not_materialized" }, { "source": "s:580", "status": "not_materialized" }, { "lowered": "l:402", "source": "s:581", "status": "whole" }, { "lowered": "l:403", "source": "s:582", "status": "whole" }, { "source": "s:583", "status": "not_materialized" }, { "source": "s:584", "status": "not_materialized" }, { "source": "s:585", "status": "not_materialized" }, { "lowered": "l:404", "source": "s:586", "status": "whole" }, { "lowered": "l:405", "source": "s:587", "status": "whole" }, { "lowered": "l:406", "source": "s:588", "status": "whole" }, { "source": "s:589", "status": "not_materialized" }, { "lowered": "l:407", "source": "s:590", "status": "whole" }, { "lowered": "l:408", "source": "s:591", "status": "whole" }, { "lowered": "l:409", "source": "s:592", "status": "whole" }, { "source": "s:593", "status": "not_materialized" }, { "lowered": "l:410", "source": "s:594", "status": "whole" }, { "lowered": "l:411", "source": "s:595", "status": "whole" }, { "lowered": "l:412", "source": "s:596", "status": "whole" }, { "lowered": "l:413", "source": "s:597", "status": "whole" }, { "lowered": "l:414", "source": "s:598", "status": "whole" }, { "source": "s:599", "status": "not_materialized" }, { "source": "s:600", "status": "not_materialized" }, { "source": "s:601", "status": "not_materialized" }, { "source": "s:602", "status": "not_materialized" }, { "lowered": "l:415", "source": "s:603", "status": "whole" }, { "lowered": "l:416", "source": "s:604", "status": "whole" }, { "lowered": "l:417", "source": "s:605", "status": "whole" }, { "lowered": "l:418", "source": "s:606", "status": "whole" }, { "source": "s:607", "status": "not_materialized" }, { "lowered": "l:419", "source": "s:608", "status": "whole" }, { "lowered": "l:420", "source": "s:609", "status": "whole" }, { "lowered": "l:421", "source": "s:610", "status": "whole" }, { "source": "s:611", "status": "not_materialized" }, { "source": "s:612", "status": "not_materialized" }, { "source": "s:613", "status": "not_materialized" }, { "source": "s:614", "status": "not_materialized" }, { "source": "s:615", "status": "not_materialized" }, { "lowered": "l:422", "source": "s:616", "status": "whole" }, { "lowered": "l:423", "source": "s:617", "status": "whole" }, { "lowered": "l:424", "source": "s:618", "status": "whole" }, { "lowered": "l:425", "source": "s:619", "status": "whole" }, { "lowered": "l:426", "source": "s:620", "status": "whole" }, { "lowered": "l:427", "source": "s:621", "status": "whole" }, { "lowered": "l:428", "source": "s:622", "status": "whole" }, { "lowered": "l:429", "source": "s:623", "status": "whole" }, { "lowered": "l:430", "source": "s:624", "status": "whole" }, { "lowered": "l:431", "source": "s:625", "status": "whole" }, { "lowered": "l:432", "source": "s:626", "status": "whole" }, { "lowered": "l:433", "source": "s:627", "status": "whole" }, { "lowered": "l:434", "source": "s:628", "status": "whole" }, { "lowered": "l:435", "source": "s:629", "status": "whole" }, { "lowered": "l:436", "source": "s:630", "status": "whole" }, { "source": "s:631", "status": "not_materialized" }, { "lowered": "l:437", "source": "s:632", "status": "whole" }, { "lowered": "l:438", "source": "s:633", "status": "whole" }, { "lowered": "l:439", "source": "s:634", "status": "whole" }, { "source": "s:635", "status": "not_materialized" }, { "source": "s:636", "status": "not_materialized" }, { "source": "s:637", "status": "not_materialized" }, { "source": "s:638", "status": "not_materialized" }, { "lowered": "l:440", "source": "s:639", "status": "whole" }, { "lowered": "l:441", "source": "s:640", "status": "whole" }, { "lowered": "l:442", "source": "s:641", "status": "whole" }, { "lowered": "l:443", "source": "s:642", "status": "whole" }, { "source": "s:643", "status": "not_materialized" }, { "lowered": "l:444", "source": "s:644", "status": "whole" }, { "lowered": "l:445", "source": "s:645", "status": "whole" }, { "lowered": "l:446", "source": "s:646", "status": "whole" }, { "source": "s:647", "status": "not_materialized" }, { "lowered": "l:447", "source": "s:648", "status": "whole" }, { "lowered": "l:448", "source": "s:649", "status": "whole" }, { "lowered": "l:449", "source": "s:650", "status": "whole" }, { "source": "s:651", "status": "not_materialized" }, { "lowered": "l:450", "source": "s:652", "status": "whole" }, { "lowered": "l:451", "source": "s:653", "status": "whole" }, { "lowered": "l:452", "source": "s:654", "status": "whole" }, { "lowered": "l:453", "source": "s:655", "status": "whole" }, { "lowered": "l:454", "source": "s:656", "status": "whole" }, { "source": "s:657", "status": "not_materialized" }, { "source": "s:658", "status": "not_materialized" }, { "source": "s:659", "status": "not_materialized" }, { "source": "s:660", "status": "not_materialized" }, { "source": "s:661", "status": "not_materialized" }, { "source": "s:662", "status": "not_materialized" }, { "source": "s:663", "status": "not_materialized" }, { "lowered": "l:455", "source": "s:664", "status": "whole" }, { "lowered": "l:456", "source": "s:665", "status": "whole" }, { "lowered": "l:457", "source": "s:666", "status": "whole" }, { "source": "s:667", "status": "not_materialized" }, { "source": "s:668", "status": "not_materialized" }, { "source": "s:669", "status": "not_materialized" }, { "source": "s:670", "status": "not_materialized" }, { "source": "s:671", "status": "not_materialized" }, { "lowered": "l:458", "source": "s:672", "status": "whole" }, { "lowered": "l:459", "source": "s:673", "status": "whole" }, { "lowered": "l:460", "source": "s:674", "status": "whole" }, { "lowered": "l:461", "source": "s:675", "status": "whole" }, { "lowered": "l:462", "source": "s:676", "status": "whole" }, { "lowered": "l:463", "source": "s:677", "status": "whole" }, { "lowered": "l:464", "source": "s:678", "status": "whole" }, { "lowered": "l:465", "source": "s:679", "status": "whole" }, { "lowered": "l:466", "source": "s:680", "status": "whole" }, { "lowered": "l:467", "source": "s:681", "status": "whole" }, { "lowered": "l:468", "source": "s:682", "status": "whole" }, { "lowered": "l:469", "source": "s:683", "status": "whole" }, { "lowered": "l:470", "source": "s:684", "status": "whole" }, { "lowered": "l:471", "source": "s:685", "status": "whole" }, { "lowered": "l:472", "source": "s:686", "status": "whole" }, { "source": "s:687", "status": "not_materialized" }, { "lowered": "l:473", "source": "s:688", "status": "whole" }, { "lowered": "l:474", "source": "s:689", "status": "whole" }, { "lowered": "l:475", "source": "s:690", "status": "whole" }, { "source": "s:691", "status": "not_materialized" }, { "source": "s:692", "status": "not_materialized" }, { "source": "s:693", "status": "not_materialized" }, { "source": "s:694", "status": "not_materialized" }, { "lowered": "l:476", "source": "s:695", "status": "whole" }, { "lowered": "l:477", "source": "s:696", "status": "whole" }, { "lowered": "l:478", "source": "s:697", "status": "whole" }, { "lowered": "l:479", "source": "s:698", "status": "whole" }, { "source": "s:699", "status": "not_materialized" }, { "lowered": "l:480", "source": "s:700", "status": "whole" }, { "lowered": "l:481", "source": "s:701", "status": "whole" }, { "lowered": "l:482", "source": "s:702", "status": "whole" }, { "source": "s:703", "status": "not_materialized" }, { "lowered": "l:483", "source": "s:704", "status": "whole" }, { "lowered": "l:484", "source": "s:705", "status": "whole" }, { "lowered": "l:485", "source": "s:706", "status": "whole" }, { "source": "s:707", "status": "not_materialized" }, { "lowered": "l:486", "source": "s:708", "status": "whole" }, { "lowered": "l:487", "source": "s:709", "status": "whole" }, { "lowered": "l:488", "source": "s:710", "status": "whole" }, { "lowered": "l:489", "source": "s:711", "status": "whole" }, { "lowered": "l:490", "source": "s:712", "status": "whole" }, { "source": "s:713", "status": "not_materialized" }, { "source": "s:714", "status": "not_materialized" }, { "source": "s:715", "status": "not_materialized" }, { "source": "s:716", "status": "not_materialized" }, { "source": "s:717", "status": "not_materialized" }, { "source": "s:718", "status": "not_materialized" }, { "source": "s:719", "status": "not_materialized" }, { "source": "s:720", "status": "not_materialized" }, { "source": "s:721", "status": "not_materialized" }, { "source": "s:722", "status": "not_materialized" }, { "source": "s:723", "status": "not_materialized" }, { "source": "s:724", "status": "not_materialized" }, { "lowered": "l:491", "source": "s:725", "status": "whole" }, { "lowered": "l:492", "source": "s:726", "status": "whole" }, { "lowered": "l:493", "source": "s:727", "status": "whole" }, { "source": "s:728", "status": "not_materialized" }, { "lowered": "l:494", "source": "s:729", "status": "whole" }, { "lowered": "l:495", "source": "s:730", "status": "whole" }, { "lowered": "l:496", "source": "s:731", "status": "whole" }, { "source": "s:732", "status": "not_materialized" }, { "source": "s:733", "status": "not_materialized" }, { "source": "s:734", "status": "not_materialized" }, { "source": "s:735", "status": "not_materialized" }, { "source": "s:736", "status": "not_materialized" }, { "source": "s:737", "status": "not_materialized" }, { "lowered": "l:497", "source": "s:738", "status": "whole" }, { "lowered": "l:498", "source": "s:739", "status": "whole" }, { "lowered": "l:499", "source": "s:740", "status": "whole" }, { "lowered": "l:500", "source": "s:741", "status": "whole" }, { "lowered": "l:501", "source": "s:742", "status": "whole" }, { "lowered": "l:502", "source": "s:743", "status": "whole" }, { "lowered": "l:503", "source": "s:744", "status": "whole" }, { "lowered": "l:504", "source": "s:745", "status": "whole" }, { "lowered": "l:505", "source": "s:746", "status": "whole" }, { "lowered": "l:506", "source": "s:747", "status": "whole" }, { "source": "s:748", "status": "not_materialized" }, { "source": "s:749", "status": "not_materialized" }, { "source": "s:750", "status": "not_materialized" }, { "lowered": "l:507", "source": "s:751", "status": "whole" }, { "lowered": "l:508", "source": "s:752", "status": "whole" }, { "lowered": "l:509", "source": "s:753", "status": "whole" }, { "source": "s:754", "status": "not_materialized" }, { "source": "s:755", "status": "not_materialized" }, { "source": "s:756", "status": "not_materialized" }, { "source": "s:757", "status": "not_materialized" }, { "lowered": "l:510", "source": "s:758", "status": "whole" }, { "lowered": "l:511", "source": "s:759", "status": "whole" }, { "lowered": "l:512", "source": "s:760", "status": "whole" }, { "lowered": "l:513", "source": "s:761", "status": "whole" }, { "source": "s:762", "status": "not_materialized" }, { "lowered": "l:514", "source": "s:763", "status": "whole" }, { "lowered": "l:515", "source": "s:764", "status": "whole" }, { "lowered": "l:516", "source": "s:765", "status": "whole" }, { "source": "s:766", "status": "not_materialized" }, { "lowered": "l:517", "source": "s:767", "status": "whole" }, { "lowered": "l:518", "source": "s:768", "status": "whole" }, { "lowered": "l:519", "source": "s:769", "status": "whole" }, { "source": "s:770", "status": "not_materialized" }, { "lowered": "l:520", "source": "s:771", "status": "whole" }, { "lowered": "l:521", "source": "s:772", "status": "whole" }, { "lowered": "l:522", "source": "s:773", "status": "whole" }, { "lowered": "l:523", "source": "s:774", "status": "whole" }, { "lowered": "l:524", "source": "s:775", "status": "whole" }, { "source": "s:776", "status": "not_materialized" }, { "source": "s:777", "status": "not_materialized" }, { "source": "s:778", "status": "not_materialized" }, { "source": "s:779", "status": "not_materialized" }, { "lowered": "l:525", "source": "s:780", "status": "whole" }, { "lowered": "l:526", "source": "s:781", "status": "whole" }, { "lowered": "l:527", "source": "s:782", "status": "whole" }, { "source": "s:783", "status": "not_materialized" }, { "lowered": "l:528", "source": "s:784", "status": "whole" }, { "source": "s:785", "status": "not_materialized" }, { "lowered": "l:529", "source": "s:786", "status": "whole" }, { "lowered": "l:530", "source": "s:787", "status": "whole" }, { "lowered": "l:531", "source": "s:788", "status": "whole" }, { "lowered": "l:532", "source": "s:789", "status": "whole" }, { "lowered": "l:533", "source": "s:790", "status": "whole" }, { "source": "s:791", "status": "not_materialized" }, { "lowered": "l:534", "source": "s:792", "status": "whole" }, { "lowered": "l:535", "source": "s:793", "status": "whole" }, { "lowered": "l:536", "source": "s:794", "status": "whole" }, { "source": "s:795", "status": "not_materialized" }, { "source": "s:796", "status": "not_materialized" }, { "source": "s:797", "status": "not_materialized" }, { "source": "s:798", "status": "not_materialized" }, { "source": "s:799", "status": "not_materialized" }, { "source": "s:800", "status": "not_materialized" }, { "source": "s:801", "status": "not_materialized" }, { "source": "s:802", "status": "not_materialized" }, { "source": "s:803", "status": "not_materialized" }, { "source": "s:804", "status": "not_materialized" }, { "source": "s:805", "status": "not_materialized" }, { "source": "s:806", "status": "not_materialized" }, { "lowered": "l:537", "source": "s:807", "status": "whole" }, { "lowered": "l:538", "source": "s:808", "status": "whole" }, { "lowered": "l:539", "source": "s:809", "status": "whole" }, { "source": "s:810", "status": "not_materialized" }, { "lowered": "l:540", "source": "s:811", "status": "whole" }, { "lowered": "l:541", "source": "s:812", "status": "whole" }, { "lowered": "l:542", "source": "s:813", "status": "whole" }, { "source": "s:814", "status": "not_materialized" }, { "source": "s:815", "status": "not_materialized" }, { "source": "s:816", "status": "not_materialized" }, { "source": "s:817", "status": "not_materialized" }, { "lowered": "l:543", "source": "s:818", "status": "whole" }, { "lowered": "l:544", "source": "s:819", "status": "whole" }, { "lowered": "l:545", "source": "s:820", "status": "whole" }, { "lowered": "l:546", "source": "s:821", "status": "whole" }, { "source": "s:822", "status": "not_materialized" }, { "lowered": "l:547", "source": "s:823", "status": "whole" }, { "lowered": "l:548", "source": "s:824", "status": "whole" }, { "lowered": "l:549", "source": "s:825", "status": "whole" }, { "source": "s:826", "status": "not_materialized" }, { "lowered": "l:550", "source": "s:827", "status": "whole" }, { "lowered": "l:551", "source": "s:828", "status": "whole" }, { "lowered": "l:552", "source": "s:829", "status": "whole" }, { "source": "s:830", "status": "not_materialized" }, { "lowered": "l:553", "source": "s:831", "status": "whole" }, { "lowered": "l:554", "source": "s:832", "status": "whole" }, { "lowered": "l:555", "source": "s:833", "status": "whole" }, { "lowered": "l:556", "source": "s:834", "status": "whole" }, { "lowered": "l:557", "source": "s:835", "status": "whole" }, { "source": "s:836", "status": "not_materialized" }, { "source": "s:837", "status": "not_materialized" }, { "source": "s:838", "status": "not_materialized" }, { "source": "s:839", "status": "not_materialized" }, { "lowered": "l:558", "source": "s:840", "status": "whole" }, { "lowered": "l:559", "source": "s:841", "status": "whole" }, { "lowered": "l:560", "source": "s:842", "status": "whole" }, { "source": "s:843", "status": "not_materialized" }, { "lowered": "l:561", "source": "s:844", "status": "whole" }, { "source": "s:845", "status": "not_materialized" }, { "lowered": "l:562", "source": "s:846", "status": "whole" }, { "lowered": "l:563", "source": "s:847", "status": "whole" }, { "lowered": "l:564", "source": "s:848", "status": "whole" }, { "lowered": "l:565", "source": "s:849", "status": "whole" }, { "lowered": "l:566", "source": "s:850", "status": "whole" }, { "source": "s:851", "status": "not_materialized" }, { "lowered": "l:567", "source": "s:852", "status": "whole" }, { "lowered": "l:568", "source": "s:853", "status": "whole" }, { "lowered": "l:569", "source": "s:854", "status": "whole" }, { "source": "s:855", "status": "not_materialized" }, { "source": "s:856", "status": "not_materialized" }, { "source": "s:857", "status": "not_materialized" }, { "source": "s:858", "status": "not_materialized" }, { "lowered": "l:570", "source": "s:859", "status": "whole" }, { "lowered": "l:571", "source": "s:860", "status": "whole" }, { "lowered": "l:572", "source": "s:861", "status": "whole" }, { "source": "s:862", "status": "not_materialized" }, { "lowered": "l:573", "source": "s:863", "status": "whole" }, { "source": "s:864", "status": "not_materialized" }, { "lowered": "l:574", "source": "s:865", "status": "whole" }, { "lowered": "l:575", "source": "s:866", "status": "whole" }, { "lowered": "l:576", "source": "s:867", "status": "whole" }, { "lowered": "l:577", "source": "s:868", "status": "whole" }, { "lowered": "l:578", "source": "s:869", "status": "whole" }, { "source": "s:870", "status": "not_materialized" }, { "lowered": "l:579", "source": "s:871", "status": "whole" }, { "lowered": "l:580", "source": "s:872", "status": "whole" }, { "lowered": "l:581", "source": "s:873", "status": "whole" }, { "source": "s:874", "status": "not_materialized" }, { "source": "s:875", "status": "not_materialized" }, { "source": "s:876", "status": "not_materialized" }, { "source": "s:877", "status": "not_materialized" }, { "lowered": "l:582", "source": "s:878", "status": "whole" }, { "lowered": "l:583", "source": "s:879", "status": "whole" }, { "lowered": "l:584", "source": "s:880", "status": "whole" }, { "lowered": "l:585", "source": "s:881", "status": "whole" }, { "source": "s:882", "status": "not_materialized" }, { "lowered": "l:586", "source": "s:883", "status": "whole" }, { "lowered": "l:587", "source": "s:884", "status": "whole" }, { "lowered": "l:588", "source": "s:885", "status": "whole" }, { "source": "s:886", "status": "not_materialized" }, { "lowered": "l:589", "source": "s:887", "status": "whole" }, { "lowered": "l:590", "source": "s:888", "status": "whole" }, { "lowered": "l:591", "source": "s:889", "status": "whole" }, { "source": "s:890", "status": "not_materialized" }, { "lowered": "l:592", "source": "s:891", "status": "whole" }, { "lowered": "l:593", "source": "s:892", "status": "whole" }, { "lowered": "l:594", "source": "s:893", "status": "whole" }, { "lowered": "l:595", "source": "s:894", "status": "whole" }, { "lowered": "l:596", "source": "s:895", "status": "whole" }, { "source": "s:896", "status": "not_materialized" }, { "source": "s:897", "status": "not_materialized" }, { "source": "s:898", "status": "not_materialized" }, { "source": "s:899", "status": "not_materialized" }, { "lowered": "l:597", "source": "s:900", "status": "whole" }, { "lowered": "l:598", "source": "s:901", "status": "whole" }, { "lowered": "l:599", "source": "s:902", "status": "whole" }, { "source": "s:903", "status": "not_materialized" }, { "lowered": "l:600", "source": "s:904", "status": "whole" }, { "source": "s:905", "status": "not_materialized" }, { "lowered": "l:601", "source": "s:906", "status": "whole" }, { "lowered": "l:602", "source": "s:907", "status": "whole" }, { "lowered": "l:603", "source": "s:908", "status": "whole" }, { "lowered": "l:604", "source": "s:909", "status": "whole" }, { "lowered": "l:605", "source": "s:910", "status": "whole" }, { "source": "s:911", "status": "not_materialized" }, { "lowered": "l:606", "source": "s:912", "status": "whole" }, { "lowered": "l:607", "source": "s:913", "status": "whole" }, { "lowered": "l:608", "source": "s:914", "status": "whole" }, { "source": "s:915", "status": "not_materialized" }, { "source": "s:916", "status": "not_materialized" }, { "source": "s:917", "status": "not_materialized" }, { "source": "s:918", "status": "not_materialized" }, { "lowered": "l:609", "source": "s:919", "status": "whole" }, { "lowered": "l:610", "source": "s:920", "status": "whole" }, { "lowered": "l:611", "source": "s:921", "status": "whole" }, { "source": "s:922", "status": "not_materialized" }, { "lowered": "l:612", "source": "s:923", "status": "whole" }, { "source": "s:924", "status": "not_materialized" }, { "lowered": "l:613", "source": "s:925", "status": "whole" }, { "lowered": "l:614", "source": "s:926", "status": "whole" }, { "lowered": "l:615", "source": "s:927", "status": "whole" }, { "lowered": "l:616", "source": "s:928", "status": "whole" }, { "lowered": "l:617", "source": "s:929", "status": "whole" }, { "source": "s:930", "status": "not_materialized" }, { "lowered": "l:618", "source": "s:931", "status": "whole" }, { "lowered": "l:619", "source": "s:932", "status": "whole" }, { "lowered": "l:620", "source": "s:933", "status": "whole" }, { "source": "s:934", "status": "not_materialized" }, { "source": "s:935", "status": "not_materialized" }, { "source": "s:936", "status": "not_materialized" }, { "source": "s:937", "status": "not_materialized" }, { "source": "s:938", "status": "not_materialized" }, { "source": "s:939", "status": "not_materialized" }, { "source": "s:940", "status": "not_materialized" }, { "lowered": "l:622", "source": "s:941", "status": "whole" }, { "lowered": "l:623", "source": "s:942", "status": "whole" }, { "lowered": "l:624", "source": "s:943", "status": "whole" }, { "source": "s:944", "status": "not_materialized" }, { "lowered": "l:625", "source": "s:945", "status": "whole" }, { "lowered": "l:626", "source": "s:946", "status": "whole" }, { "lowered": "l:627", "source": "s:947", "status": "whole" }, { "source": "s:948", "status": "not_materialized" }, { "lowered": "l:628", "source": "s:949", "status": "whole" }, { "lowered": "l:629", "source": "s:950", "status": "whole" }, { "lowered": "l:630", "source": "s:951", "status": "whole" }, { "lowered": "l:631", "source": "s:952", "status": "whole" }, { "lowered": "l:632", "source": "s:953", "status": "whole" }, { "lowered": "l:633", "source": "s:954", "status": "whole" }, { "lowered": "l:634", "source": "s:955", "status": "whole" }, { "lowered": "l:635", "source": "s:956", "status": "whole" }, { "lowered": "l:636", "source": "s:957", "status": "whole" }, { "lowered": "l:637", "source": "s:958", "status": "whole" }, { "lowered": "l:638", "source": "s:959", "status": "whole" }, { "source": "s:960", "status": "not_materialized" }, { "source": "s:961", "status": "not_materialized" }, { "source": "s:962", "status": "not_materialized" }, { "source": "s:963", "status": "not_materialized" }, { "source": "s:964", "status": "not_materialized" }, { "source": "s:965", "status": "not_materialized" }, { "source": "s:966", "status": "not_materialized" }, { "source": "s:967", "status": "not_materialized" }, { "source": "s:968", "status": "not_materialized" }, { "source": "s:969", "status": "not_materialized" }, { "source": "s:970", "status": "not_materialized" }, { "source": "s:971", "status": "not_materialized" }, { "lowered": "l:639", "source": "s:972", "status": "whole" }, { "lowered": "l:640", "source": "s:973", "status": "whole" }, { "lowered": "l:641", "source": "s:974", "status": "whole" }, { "source": "s:975", "status": "not_materialized" }, { "lowered": "l:642", "source": "s:976", "status": "whole" }, { "lowered": "l:643", "source": "s:977", "status": "whole" }, { "lowered": "l:644", "source": "s:978", "status": "whole" }, { "source": "s:979", "status": "not_materialized" }, { "lowered": "l:645", "source": "s:980", "status": "whole" }, { "lowered": "l:646", "source": "s:981", "status": "whole" }, { "lowered": "l:647", "source": "s:982", "status": "whole" }, { "source": "s:983", "status": "not_materialized" }, { "lowered": "l:648", "source": "s:984", "status": "whole" }, { "lowered": "l:649", "source": "s:985", "status": "whole" }, { "lowered": "l:650", "source": "s:986", "status": "whole" }, { "lowered": "l:651", "source": "s:987", "status": "whole" }, { "lowered": "l:652", "source": "s:988", "status": "whole" }, { "lowered": "l:653", "source": "s:989", "status": "whole" }, { "lowered": "l:654", "source": "s:990", "status": "whole" }, { "source": "s:991", "status": "not_materialized" }, { "source": "s:992", "status": "not_materialized" }, { "source": "s:993", "status": "not_materialized" }, { "lowered": "l:658", "source": "s:994", "status": "whole" }, { "lowered": "l:659", "source": "s:995", "status": "whole" }, { "lowered": "l:660", "source": "s:996", "status": "whole" }, { "source": "s:997", "status": "not_materialized" }, { "lowered": "l:655", "source": "s:998", "status": "whole" }, { "lowered": "l:656", "source": "s:999", "status": "whole" }, { "lowered": "l:657", "source": "s:1000", "status": "whole" }, { "source": "s:1001", "status": "not_materialized" }, { "source": "s:1002", "status": "not_materialized" }, { "source": "s:1003", "status": "not_materialized" }, { "source": "s:1004", "status": "not_materialized" }, { "lowered": "l:661", "source": "s:1005", "status": "whole" }, { "source": "s:1006", "status": "not_materialized" }, { "lowered": "l:662", "source": "s:1007", "status": "whole" }, { "lowered": "l:663", "source": "s:1008", "status": "whole" }, { "lowered": "l:664", "source": "s:1009", "status": "whole" }, { "lowered": "l:665", "source": "s:1010", "status": "whole" }, { "lowered": "l:666", "source": "s:1011", "status": "whole" }, { "lowered": "l:667", "source": "s:1012", "status": "whole" }, { "lowered": "l:668", "source": "s:1013", "status": "whole" }, { "lowered": "l:669", "source": "s:1014", "status": "whole" }, { "lowered": "l:670", "source": "s:1015", "status": "whole" }, { "lowered": "l:671", "source": "s:1016", "status": "whole" }, { "lowered": "l:672", "source": "s:1017", "status": "whole" }, { "lowered": "l:673", "source": "s:1018", "status": "whole" }, { "lowered": "l:674", "source": "s:1019", "status": "whole" }, { "lowered": "l:675", "source": "s:1020", "status": "whole" }, { "source": "s:1021", "status": "not_materialized" }, { "source": "s:1022", "status": "not_materialized" }, { "source": "s:1023", "status": "not_materialized" }, { "lowered": "l:676", "source": "s:1024", "status": "whole" }, { "lowered": "l:677", "source": "s:1025", "status": "whole" }, { "source": "s:1026", "status": "not_materialized" }, { "lowered": "l:678", "source": "s:1027", "status": "whole" }, { "lowered": "l:679", "source": "s:1028", "status": "whole" }, { "lowered": "l:680", "source": "s:1029", "status": "whole" }, { "lowered": "l:681", "source": "s:1030", "status": "whole" }, { "lowered": "l:682", "source": "s:1031", "status": "whole" }, { "lowered": "l:683", "source": "s:1032", "status": "whole" }, { "source": "s:1033", "status": "not_materialized" }, { "source": "s:1034", "status": "not_materialized" }, { "source": "s:1035", "status": "not_materialized" }, { "source": "s:1036", "status": "not_materialized" }, { "lowered": "l:684", "source": "s:1037", "status": "whole" }, { "source": "s:1038", "status": "not_materialized" }, { "lowered": "l:685", "source": "s:1039", "status": "whole" }, { "lowered": "l:686", "source": "s:1040", "status": "whole" }, { "lowered": "l:687", "source": "s:1041", "status": "whole" }, { "lowered": "l:688", "source": "s:1042", "status": "whole" }, { "lowered": "l:689", "source": "s:1043", "status": "whole" }, { "lowered": "l:690", "source": "s:1044", "status": "whole" }, { "lowered": "l:691", "source": "s:1045", "status": "whole" }, { "lowered": "l:692", "source": "s:1046", "status": "whole" }, { "source": "s:1047", "status": "not_materialized" }, { "source": "s:1048", "status": "not_materialized" }, { "source": "s:1049", "status": "not_materialized" }, { "source": "s:1050", "status": "not_materialized" }, { "lowered": "l:693", "source": "s:1051", "status": "whole" }, { "lowered": "l:694", "source": "s:1052", "status": "whole" }, { "lowered": "l:695", "source": "s:1053", "status": "whole" }, { "source": "s:1054", "status": "not_materialized" }, { "lowered": "l:696", "source": "s:1055", "status": "whole" }, { "source": "s:1056", "status": "not_materialized" }, { "lowered": "l:697", "source": "s:1057", "status": "whole" }, { "lowered": "l:698", "source": "s:1058", "status": "whole" }, { "lowered": "l:699", "source": "s:1059", "status": "whole" }, { "lowered": "l:700", "source": "s:1060", "status": "whole" }, { "lowered": "l:701", "source": "s:1061", "status": "whole" }, { "source": "s:1062", "status": "not_materialized" }, { "lowered": "l:702", "source": "s:1063", "status": "whole" }, { "lowered": "l:703", "source": "s:1064", "status": "whole" }, { "lowered": "l:704", "source": "s:1065", "status": "whole" }, { "source": "s:1066", "status": "not_materialized" }, { "lowered": "l:705", "source": "s:1067", "status": "whole" }, { "lowered": "l:706", "source": "s:1068", "status": "whole" }, { "lowered": "l:707", "source": "s:1069", "status": "whole" }, { "source": "s:1070", "status": "not_materialized" }, { "lowered": "l:708", "source": "s:1071", "status": "whole" }, { "lowered": "l:709", "source": "s:1072", "status": "whole" }, { "lowered": "l:710", "source": "s:1073", "status": "whole" }, { "lowered": "l:711", "source": "s:1074", "status": "whole" }, { "lowered": "l:712", "source": "s:1075", "status": "whole" }, { "lowered": "l:713", "source": "s:1076", "status": "whole" }, { "lowered": "l:714", "source": "s:1077", "status": "whole" }, { "source": "s:1078", "status": "not_materialized" }, { "source": "s:1079", "status": "not_materialized" }, { "source": "s:1080", "status": "not_materialized" }, { "lowered": "l:715", "source": "s:1081", "status": "whole" }, { "lowered": "l:716", "source": "s:1082", "status": "whole" }, { "lowered": "l:717", "source": "s:1083", "status": "whole" }, { "source": "s:1084", "status": "not_materialized" }, { "lowered": "l:718", "source": "s:1085", "status": "whole" }, { "lowered": "l:719", "source": "s:1086", "status": "whole" }, { "source": "s:1087", "status": "not_materialized" }, { "source": "s:1088", "status": "not_materialized" }, { "source": "s:1089", "status": "not_materialized" }, { "source": "s:1090", "status": "not_materialized" }, { "lowered": "l:720", "source": "s:1091", "status": "whole" }, { "lowered": "l:721", "source": "s:1092", "status": "whole" }, { "lowered": "l:722", "source": "s:1093", "status": "whole" }, { "lowered": "l:723", "source": "s:1094", "status": "whole" }, { "lowered": "l:724", "source": "s:1095", "status": "whole" }, { "lowered": "l:725", "source": "s:1096", "status": "whole" }, { "lowered": "l:726", "source": "s:1097", "status": "whole" }, { "source": "s:1098", "status": "not_materialized" }, { "lowered": "l:727", "source": "s:1099", "status": "whole" }, { "lowered": "l:728", "source": "s:1100", "status": "whole" }, { "lowered": "l:729", "source": "s:1101", "status": "whole" }, { "lowered": "l:730", "source": "s:1102", "status": "whole" }, { "lowered": "l:731", "source": "s:1103", "status": "whole" }, { "lowered": "l:732", "source": "s:1104", "status": "whole" }, { "source": "s:1105", "status": "not_materialized" }, { "source": "s:1106", "status": "not_materialized" }, { "source": "s:1107", "status": "not_materialized" }, { "source": "s:1108", "status": "not_materialized" }, { "lowered": "l:733", "source": "s:1109", "status": "whole" }, { "lowered": "l:734", "source": "s:1110", "status": "whole" }, { "lowered": "l:735", "source": "s:1111", "status": "whole" }, { "lowered": "l:736", "source": "s:1112", "status": "whole" }, { "lowered": "l:737", "source": "s:1113", "status": "whole" }, { "lowered": "l:738", "source": "s:1114", "status": "whole" }, { "source": "s:1115", "status": "not_materialized" }, { "source": "s:1116", "status": "not_materialized" }, { "source": "s:1117", "status": "not_materialized" }, { "source": "s:1118", "status": "not_materialized" }, { "lowered": "l:739", "source": "s:1119", "status": "whole" }, { "lowered": "l:740", "source": "s:1120", "status": "whole" }, { "lowered": "l:741", "source": "s:1121", "status": "whole" }, { "source": "s:1122", "status": "not_materialized" }, { "source": "s:1123", "status": "not_materialized" }, { "source": "s:1124", "status": "not_materialized" }, { "source": "s:1125", "status": "not_materialized" }, { "lowered": "l:742", "source": "s:1126", "status": "whole" }, { "lowered": "l:743", "source": "s:1127", "status": "whole" }, { "lowered": "l:744", "source": "s:1128", "status": "whole" }, { "source": "s:1129", "status": "not_materialized" }, { "lowered": "l:745", "source": "s:1130", "status": "whole" }, { "source": "s:1131", "status": "not_materialized" }, { "lowered": "l:746", "source": "s:1132", "status": "whole" }, { "lowered": "l:747", "source": "s:1133", "status": "whole" }, { "lowered": "l:748", "source": "s:1134", "status": "whole" }, { "lowered": "l:749", "source": "s:1135", "status": "whole" }, { "lowered": "l:750", "source": "s:1136", "status": "whole" }, { "source": "s:1137", "status": "not_materialized" }, { "lowered": "l:751", "source": "s:1138", "status": "whole" }, { "lowered": "l:752", "source": "s:1139", "status": "whole" }, { "lowered": "l:753", "source": "s:1140", "status": "whole" }, { "source": "s:1141", "status": "not_materialized" }, { "lowered": "l:754", "source": "s:1142", "status": "whole" }, { "lowered": "l:755", "source": "s:1143", "status": "whole" }, { "lowered": "l:756", "source": "s:1144", "status": "whole" }, { "source": "s:1145", "status": "not_materialized" }, { "lowered": "l:757", "source": "s:1146", "status": "whole" }, { "lowered": "l:758", "source": "s:1147", "status": "whole" }, { "lowered": "l:759", "source": "s:1148", "status": "whole" }, { "lowered": "l:760", "source": "s:1149", "status": "whole" }, { "lowered": "l:761", "source": "s:1150", "status": "whole" }, { "lowered": "l:762", "source": "s:1151", "status": "whole" }, { "lowered": "l:763", "source": "s:1152", "status": "whole" }, { "source": "s:1153", "status": "not_materialized" }, { "source": "s:1154", "status": "not_materialized" }, { "source": "s:1155", "status": "not_materialized" }, { "lowered": "l:764", "source": "s:1156", "status": "whole" }, { "lowered": "l:765", "source": "s:1157", "status": "whole" }, { "lowered": "l:766", "source": "s:1158", "status": "whole" }, { "source": "s:1159", "status": "not_materialized" }, { "lowered": "l:767", "source": "s:1160", "status": "whole" }, { "lowered": "l:768", "source": "s:1161", "status": "whole" }, { "source": "s:1162", "status": "not_materialized" }, { "source": "s:1163", "status": "not_materialized" }, { "source": "s:1164", "status": "not_materialized" }, { "source": "s:1165", "status": "not_materialized" }, { "lowered": "l:769", "source": "s:1166", "status": "whole" }, { "lowered": "l:770", "source": "s:1167", "status": "whole" }, { "lowered": "l:771", "source": "s:1168", "status": "whole" }, { "lowered": "l:772", "source": "s:1169", "status": "whole" }, { "lowered": "l:773", "source": "s:1170", "status": "whole" }, { "lowered": "l:774", "source": "s:1171", "status": "whole" }, { "lowered": "l:775", "source": "s:1172", "status": "whole" }, { "source": "s:1173", "status": "not_materialized" }, { "lowered": "l:776", "source": "s:1174", "status": "whole" }, { "lowered": "l:777", "source": "s:1175", "status": "whole" }, { "lowered": "l:778", "source": "s:1176", "status": "whole" }, { "lowered": "l:779", "source": "s:1177", "status": "whole" }, { "lowered": "l:780", "source": "s:1178", "status": "whole" }, { "lowered": "l:781", "source": "s:1179", "status": "whole" }, { "source": "s:1180", "status": "not_materialized" }, { "source": "s:1181", "status": "not_materialized" }, { "source": "s:1182", "status": "not_materialized" }, { "source": "s:1183", "status": "not_materialized" }, { "lowered": "l:782", "source": "s:1184", "status": "whole" }, { "lowered": "l:783", "source": "s:1185", "status": "whole" }, { "lowered": "l:784", "source": "s:1186", "status": "whole" }, { "lowered": "l:785", "source": "s:1187", "status": "whole" }, { "lowered": "l:786", "source": "s:1188", "status": "whole" }, { "source": "s:1189", "status": "not_materialized" }, { "source": "s:1190", "status": "not_materialized" }, { "source": "s:1191", "status": "not_materialized" }, { "source": "s:1192", "status": "not_materialized" }, { "lowered": "l:787", "source": "s:1193", "status": "whole" }, { "lowered": "l:788", "source": "s:1194", "status": "whole" }, { "lowered": "l:789", "source": "s:1195", "status": "whole" }, { "source": "s:1196", "status": "not_materialized" }, { "source": "s:1197", "status": "not_materialized" }, { "source": "s:1198", "status": "not_materialized" }, { "source": "s:1199", "status": "not_materialized" }, { "lowered": "l:790", "source": "s:1200", "status": "whole" }, { "lowered": "l:791", "source": "s:1201", "status": "whole" }, { "lowered": "l:792", "source": "s:1202", "status": "whole" }, { "source": "s:1203", "status": "not_materialized" }, { "lowered": "l:793", "source": "s:1204", "status": "whole" }, { "source": "s:1205", "status": "not_materialized" }, { "lowered": "l:794", "source": "s:1206", "status": "whole" }, { "lowered": "l:795", "source": "s:1207", "status": "whole" }, { "lowered": "l:796", "source": "s:1208", "status": "whole" }, { "lowered": "l:797", "source": "s:1209", "status": "whole" }, { "lowered": "l:798", "source": "s:1210", "status": "whole" }, { "source": "s:1211", "status": "not_materialized" }, { "lowered": "l:799", "source": "s:1212", "status": "whole" }, { "lowered": "l:800", "source": "s:1213", "status": "whole" }, { "lowered": "l:801", "source": "s:1214", "status": "whole" }, { "source": "s:1215", "status": "not_materialized" }, { "lowered": "l:802", "source": "s:1216", "status": "whole" }, { "lowered": "l:803", "source": "s:1217", "status": "whole" }, { "lowered": "l:804", "source": "s:1218", "status": "whole" }, { "source": "s:1219", "status": "not_materialized" }, { "lowered": "l:805", "source": "s:1220", "status": "whole" }, { "lowered": "l:806", "source": "s:1221", "status": "whole" }, { "lowered": "l:807", "source": "s:1222", "status": "whole" }, { "lowered": "l:808", "source": "s:1223", "status": "whole" }, { "lowered": "l:809", "source": "s:1224", "status": "whole" }, { "lowered": "l:810", "source": "s:1225", "status": "whole" }, { "lowered": "l:811", "source": "s:1226", "status": "whole" }, { "source": "s:1227", "status": "not_materialized" }, { "source": "s:1228", "status": "not_materialized" }, { "source": "s:1229", "status": "not_materialized" }, { "lowered": "l:812", "source": "s:1230", "status": "whole" }, { "lowered": "l:813", "source": "s:1231", "status": "whole" }, { "lowered": "l:814", "source": "s:1232", "status": "whole" }, { "source": "s:1233", "status": "not_materialized" }, { "lowered": "l:815", "source": "s:1234", "status": "whole" }, { "lowered": "l:816", "source": "s:1235", "status": "whole" }, { "source": "s:1236", "status": "not_materialized" }, { "source": "s:1237", "status": "not_materialized" }, { "source": "s:1238", "status": "not_materialized" }, { "source": "s:1239", "status": "not_materialized" }, { "lowered": "l:817", "source": "s:1240", "status": "whole" }, { "lowered": "l:818", "source": "s:1241", "status": "whole" }, { "lowered": "l:819", "source": "s:1242", "status": "whole" }, { "lowered": "l:820", "source": "s:1243", "status": "whole" }, { "lowered": "l:821", "source": "s:1244", "status": "whole" }, { "lowered": "l:822", "source": "s:1245", "status": "whole" }, { "lowered": "l:823", "source": "s:1246", "status": "whole" }, { "source": "s:1247", "status": "not_materialized" }, { "lowered": "l:824", "source": "s:1248", "status": "whole" }, { "lowered": "l:825", "source": "s:1249", "status": "whole" }, { "lowered": "l:826", "source": "s:1250", "status": "whole" }, { "lowered": "l:827", "source": "s:1251", "status": "whole" }, { "lowered": "l:828", "source": "s:1252", "status": "whole" }, { "lowered": "l:829", "source": "s:1253", "status": "whole" }, { "source": "s:1254", "status": "not_materialized" }, { "source": "s:1255", "status": "not_materialized" }, { "source": "s:1256", "status": "not_materialized" }, { "source": "s:1257", "status": "not_materialized" }, { "lowered": "l:830", "source": "s:1258", "status": "whole" }, { "lowered": "l:831", "source": "s:1259", "status": "whole" }, { "lowered": "l:832", "source": "s:1260", "status": "whole" }, { "lowered": "l:833", "source": "s:1261", "status": "whole" }, { "lowered": "l:834", "source": "s:1262", "status": "whole" }, { "source": "s:1263", "status": "not_materialized" }, { "source": "s:1264", "status": "not_materialized" }, { "source": "s:1265", "status": "not_materialized" }, { "source": "s:1266", "status": "not_materialized" }, { "lowered": "l:835", "source": "s:1267", "status": "whole" }, { "lowered": "l:836", "source": "s:1268", "status": "whole" }, { "lowered": "l:837", "source": "s:1269", "status": "whole" }, { "source": "s:1270", "status": "not_materialized" }, { "source": "s:1271", "status": "not_materialized" }, { "source": "s:1272", "status": "not_materialized" }, { "source": "s:1273", "status": "not_materialized" }, { "lowered": "l:838", "source": "s:1274", "status": "whole" }, { "lowered": "l:839", "source": "s:1275", "status": "whole" }, { "lowered": "l:840", "source": "s:1276", "status": "whole" }, { "source": "s:1277", "status": "not_materialized" }, { "lowered": "l:841", "source": "s:1278", "status": "whole" }, { "source": "s:1279", "status": "not_materialized" }, { "lowered": "l:842", "source": "s:1280", "status": "whole" }, { "lowered": "l:843", "source": "s:1281", "status": "whole" }, { "lowered": "l:844", "source": "s:1282", "status": "whole" }, { "lowered": "l:845", "source": "s:1283", "status": "whole" }, { "lowered": "l:846", "source": "s:1284", "status": "whole" }, { "source": "s:1285", "status": "not_materialized" }, { "lowered": "l:847", "source": "s:1286", "status": "whole" }, { "lowered": "l:848", "source": "s:1287", "status": "whole" }, { "lowered": "l:849", "source": "s:1288", "status": "whole" }, { "source": "s:1289", "status": "not_materialized" }, { "lowered": "l:850", "source": "s:1290", "status": "whole" }, { "lowered": "l:851", "source": "s:1291", "status": "whole" }, { "lowered": "l:852", "source": "s:1292", "status": "whole" }, { "source": "s:1293", "status": "not_materialized" }, { "lowered": "l:853", "source": "s:1294", "status": "whole" }, { "lowered": "l:854", "source": "s:1295", "status": "whole" }, { "lowered": "l:855", "source": "s:1296", "status": "whole" }, { "lowered": "l:856", "source": "s:1297", "status": "whole" }, { "lowered": "l:857", "source": "s:1298", "status": "whole" }, { "lowered": "l:858", "source": "s:1299", "status": "whole" }, { "lowered": "l:859", "source": "s:1300", "status": "whole" }, { "lowered": "l:860", "source": "s:1301", "status": "whole" }, { "lowered": "l:861", "source": "s:1302", "status": "whole" }, { "lowered": "l:862", "source": "s:1303", "status": "whole" }, { "lowered": "l:863", "source": "s:1304", "status": "whole" }, { "source": "s:1305", "status": "not_materialized" }, { "source": "s:1306", "status": "not_materialized" }, { "source": "s:1307", "status": "not_materialized" }, { "source": "s:1308", "status": "not_materialized" }, { "lowered": "l:864", "source": "s:1309", "status": "whole" }, { "lowered": "l:865", "source": "s:1310", "status": "whole" }, { "lowered": "l:866", "source": "s:1311", "status": "whole" }, { "source": "s:1312", "status": "not_materialized" }, { "lowered": "l:867", "source": "s:1313", "status": "whole" }, { "source": "s:1314", "status": "not_materialized" }, { "lowered": "l:868", "source": "s:1315", "status": "whole" }, { "lowered": "l:869", "source": "s:1316", "status": "whole" }, { "lowered": "l:870", "source": "s:1317", "status": "whole" }, { "lowered": "l:871", "source": "s:1318", "status": "whole" }, { "lowered": "l:872", "source": "s:1319", "status": "whole" }, { "source": "s:1320", "status": "not_materialized" }, { "lowered": "l:873", "source": "s:1321", "status": "whole" }, { "lowered": "l:874", "source": "s:1322", "status": "whole" }, { "lowered": "l:875", "source": "s:1323", "status": "whole" }, { "source": "s:1324", "status": "not_materialized" }, { "lowered": "l:876", "source": "s:1325", "status": "whole" }, { "lowered": "l:877", "source": "s:1326", "status": "whole" }, { "lowered": "l:878", "source": "s:1327", "status": "whole" }, { "source": "s:1328", "status": "not_materialized" }, { "lowered": "l:879", "source": "s:1329", "status": "whole" }, { "lowered": "l:880", "source": "s:1330", "status": "whole" }, { "lowered": "l:881", "source": "s:1331", "status": "whole" }, { "lowered": "l:882", "source": "s:1332", "status": "whole" }, { "lowered": "l:883", "source": "s:1333", "status": "whole" }, { "lowered": "l:884", "source": "s:1334", "status": "whole" }, { "lowered": "l:885", "source": "s:1335", "status": "whole" }, { "lowered": "l:886", "source": "s:1336", "status": "whole" }, { "lowered": "l:887", "source": "s:1337", "status": "whole" }, { "lowered": "l:888", "source": "s:1338", "status": "whole" }, { "lowered": "l:889", "source": "s:1339", "status": "whole" }, { "lowered": "l:890", "source": "s:1340", "status": "whole" }, { "lowered": "l:891", "source": "s:1341", "status": "whole" }, { "lowered": "l:892", "source": "s:1342", "status": "whole" }, { "source": "s:1343", "status": "not_materialized" }, { "lowered": "l:893", "source": "s:1344", "status": "whole" }, { "lowered": "l:894", "source": "s:1345", "status": "whole" }, { "lowered": "l:895", "source": "s:1346", "status": "whole" }, { "source": "s:1347", "status": "not_materialized" }, { "lowered": "l:896", "source": "s:1348", "status": "whole" }, { "lowered": "l:897", "source": "s:1349", "status": "whole" }, { "lowered": "l:898", "source": "s:1350", "status": "whole" }, { "lowered": "l:899", "source": "s:1351", "status": "whole" }, { "lowered": "l:900", "source": "s:1352", "status": "whole" }, { "lowered": "l:901", "source": "s:1353", "status": "whole" }, { "lowered": "l:902", "source": "s:1354", "status": "whole" }, { "lowered": "l:903", "source": "s:1355", "status": "whole" }, { "lowered": "l:904", "source": "s:1356", "status": "whole" }, { "lowered": "l:905", "source": "s:1357", "status": "whole" }, { "lowered": "l:906", "source": "s:1358", "status": "whole" }, { "lowered": "l:907", "source": "s:1359", "status": "whole" }, { "lowered": "l:908", "source": "s:1360", "status": "whole" }, { "source": "s:1361", "status": "not_materialized" }, { "source": "s:1362", "status": "not_materialized" }, { "source": "s:1363", "status": "not_materialized" }, { "lowered": "l:912", "source": "s:1364", "status": "whole" }, { "lowered": "l:913", "source": "s:1365", "status": "whole" }, { "lowered": "l:914", "source": "s:1366", "status": "whole" }, { "source": "s:1367", "status": "not_materialized" }, { "lowered": "l:909", "source": "s:1368", "status": "whole" }, { "lowered": "l:910", "source": "s:1369", "status": "whole" }, { "lowered": "l:911", "source": "s:1370", "status": "whole" }, { "source": "s:1371", "status": "not_materialized" }, { "source": "s:1372", "status": "not_materialized" }, { "source": "s:1373", "status": "not_materialized" }, { "source": "s:1374", "status": "not_materialized" }, { "lowered": "l:915", "source": "s:1375", "status": "whole" }, { "source": "s:1376", "status": "not_materialized" }, { "lowered": "l:916", "source": "s:1377", "status": "whole" }, { "lowered": "l:917", "source": "s:1378", "status": "whole" }, { "lowered": "l:918", "source": "s:1379", "status": "whole" }, { "lowered": "l:919", "source": "s:1380", "status": "whole" }, { "lowered": "l:920", "source": "s:1381", "status": "whole" }, { "lowered": "l:921", "source": "s:1382", "status": "whole" }, { "lowered": "l:922", "source": "s:1383", "status": "whole" }, { "source": "s:1384", "status": "not_materialized" }, { "lowered": "l:923", "source": "s:1385", "status": "whole" }, { "lowered": "l:924", "source": "s:1386", "status": "whole" }, { "lowered": "l:925", "source": "s:1387", "status": "whole" }, { "source": "s:1388", "status": "not_materialized" }, { "lowered": "l:926", "source": "s:1389", "status": "whole" }, { "lowered": "l:927", "source": "s:1390", "status": "whole" }, { "lowered": "l:928", "source": "s:1391", "status": "whole" }, { "lowered": "l:929", "source": "s:1392", "status": "whole" }, { "lowered": "l:930", "source": "s:1393", "status": "whole" }, { "lowered": "l:931", "source": "s:1394", "status": "whole" }, { "lowered": "l:932", "source": "s:1395", "status": "whole" }, { "lowered": "l:933", "source": "s:1396", "status": "whole" }, { "lowered": "l:934", "source": "s:1397", "status": "whole" }, { "lowered": "l:935", "source": "s:1398", "status": "whole" }, { "lowered": "l:936", "source": "s:1399", "status": "whole" }, { "lowered": "l:937", "source": "s:1400", "status": "whole" }, { "lowered": "l:938", "source": "s:1401", "status": "whole" }, { "source": "s:1402", "status": "not_materialized" }, { "source": "s:1403", "status": "not_materialized" }, { "source": "s:1404", "status": "not_materialized" }, { "lowered": "l:939", "source": "s:1405", "status": "whole" }, { "lowered": "l:940", "source": "s:1406", "status": "whole" }, { "lowered": "l:941", "source": "s:1407", "status": "whole" }, { "source": "s:1408", "status": "not_materialized" }, { "lowered": "l:942", "source": "s:1409", "status": "whole" }, { "lowered": "l:943", "source": "s:1410", "status": "whole" }, { "source": "s:1411", "status": "not_materialized" }, { "source": "s:1412", "status": "not_materialized" }, { "source": "s:1413", "status": "not_materialized" }, { "source": "s:1414", "status": "not_materialized" }, { "lowered": "l:944", "source": "s:1415", "status": "whole" }, { "lowered": "l:945", "source": "s:1416", "status": "whole" }, { "lowered": "l:946", "source": "s:1417", "status": "whole" }, { "source": "s:1418", "status": "not_materialized" }, { "lowered": "l:947", "source": "s:1419", "status": "whole" }, { "lowered": "l:948", "source": "s:1420", "status": "whole" }, { "lowered": "l:949", "source": "s:1421", "status": "whole" }, { "source": "s:1422", "status": "not_materialized" }, { "lowered": "l:950", "source": "s:1423", "status": "whole" }, { "lowered": "l:951", "source": "s:1424", "status": "whole" }, { "lowered": "l:952", "source": "s:1425", "status": "whole" }, { "lowered": "l:953", "source": "s:1426", "status": "whole" }, { "lowered": "l:954", "source": "s:1427", "status": "whole" }, { "lowered": "l:955", "source": "s:1428", "status": "whole" }, { "lowered": "l:956", "source": "s:1429", "status": "whole" }, { "lowered": "l:957", "source": "s:1430", "status": "whole" }, { "lowered": "l:958", "source": "s:1431", "status": "whole" }, { "lowered": "l:959", "source": "s:1432", "status": "whole" }, { "lowered": "l:960", "source": "s:1433", "status": "whole" }, { "lowered": "l:961", "source": "s:1434", "status": "whole" }, { "lowered": "l:962", "source": "s:1435", "status": "whole" }, { "source": "s:1436", "status": "not_materialized" }, { "source": "s:1437", "status": "not_materialized" }, { "source": "s:1438", "status": "not_materialized" }, { "lowered": "l:963", "source": "s:1439", "status": "whole" }, { "lowered": "l:964", "source": "s:1440", "status": "whole" }, { "lowered": "l:965", "source": "s:1441", "status": "whole" }, { "source": "s:1442", "status": "not_materialized" }, { "lowered": "l:966", "source": "s:1443", "status": "whole" }, { "lowered": "l:967", "source": "s:1444", "status": "whole" }, { "source": "s:1445", "status": "not_materialized" }, { "source": "s:1446", "status": "not_materialized" }, { "source": "s:1447", "status": "not_materialized" }, { "source": "s:1448", "status": "not_materialized" }, { "lowered": "l:968", "source": "s:1449", "status": "whole" }, { "lowered": "l:969", "source": "s:1450", "status": "whole" }, { "lowered": "l:970", "source": "s:1451", "status": "whole" }, { "source": "s:1452", "status": "not_materialized" }, { "lowered": "l:971", "source": "s:1453", "status": "whole" }, { "lowered": "l:972", "source": "s:1454", "status": "whole" }, { "lowered": "l:973", "source": "s:1455", "status": "whole" }, { "source": "s:1456", "status": "not_materialized" }, { "lowered": "l:974", "source": "s:1457", "status": "whole" }, { "lowered": "l:975", "source": "s:1458", "status": "whole" }, { "lowered": "l:976", "source": "s:1459", "status": "whole" }, { "lowered": "l:977", "source": "s:1460", "status": "whole" }, { "lowered": "l:978", "source": "s:1461", "status": "whole" }, { "lowered": "l:979", "source": "s:1462", "status": "whole" }, { "lowered": "l:980", "source": "s:1463", "status": "whole" }, { "lowered": "l:981", "source": "s:1464", "status": "whole" }, { "lowered": "l:982", "source": "s:1465", "status": "whole" }, { "lowered": "l:983", "source": "s:1466", "status": "whole" }, { "lowered": "l:984", "source": "s:1467", "status": "whole" }, { "lowered": "l:985", "source": "s:1468", "status": "whole" }, { "lowered": "l:986", "source": "s:1469", "status": "whole" }, { "source": "s:1470", "status": "not_materialized" }, { "source": "s:1471", "status": "not_materialized" }, { "source": "s:1472", "status": "not_materialized" }, { "lowered": "l:987", "source": "s:1473", "status": "whole" }, { "lowered": "l:988", "source": "s:1474", "status": "whole" }, { "lowered": "l:989", "source": "s:1475", "status": "whole" }, { "source": "s:1476", "status": "not_materialized" }, { "lowered": "l:990", "source": "s:1477", "status": "whole" }, { "lowered": "l:991", "source": "s:1478", "status": "whole" }, { "source": "s:1479", "status": "not_materialized" }, { "source": "s:1480", "status": "not_materialized" }, { "source": "s:1481", "status": "not_materialized" }, { "source": "s:1482", "status": "not_materialized" }, { "lowered": "l:992", "source": "s:1483", "status": "whole" }, { "lowered": "l:993", "source": "s:1484", "status": "whole" }, { "lowered": "l:994", "source": "s:1485", "status": "whole" }, { "source": "s:1486", "status": "not_materialized" }, { "lowered": "l:995", "source": "s:1487", "status": "whole" }, { "lowered": "l:996", "source": "s:1488", "status": "whole" }, { "lowered": "l:997", "source": "s:1489", "status": "whole" }, { "source": "s:1490", "status": "not_materialized" }, { "lowered": "l:998", "source": "s:1491", "status": "whole" }, { "lowered": "l:999", "source": "s:1492", "status": "whole" }, { "lowered": "l:1000", "source": "s:1493", "status": "whole" }, { "lowered": "l:1001", "source": "s:1494", "status": "whole" }, { "lowered": "l:1002", "source": "s:1495", "status": "whole" }, { "lowered": "l:1003", "source": "s:1496", "status": "whole" }, { "lowered": "l:1004", "source": "s:1497", "status": "whole" }, { "lowered": "l:1005", "source": "s:1498", "status": "whole" }, { "lowered": "l:1006", "source": "s:1499", "status": "whole" }, { "lowered": "l:1007", "source": "s:1500", "status": "whole" }, { "lowered": "l:1008", "source": "s:1501", "status": "whole" }, { "lowered": "l:1009", "source": "s:1502", "status": "whole" }, { "lowered": "l:1010", "source": "s:1503", "status": "whole" }, { "source": "s:1504", "status": "not_materialized" }, { "source": "s:1505", "status": "not_materialized" }, { "source": "s:1506", "status": "not_materialized" }, { "lowered": "l:1011", "source": "s:1507", "status": "whole" }, { "lowered": "l:1012", "source": "s:1508", "status": "whole" }, { "lowered": "l:1013", "source": "s:1509", "status": "whole" }, { "source": "s:1510", "status": "not_materialized" }, { "lowered": "l:1014", "source": "s:1511", "status": "whole" }, { "lowered": "l:1015", "source": "s:1512", "status": "whole" }, { "source": "s:1513", "status": "not_materialized" }, { "source": "s:1514", "status": "not_materialized" }, { "source": "s:1515", "status": "not_materialized" }, { "source": "s:1516", "status": "not_materialized" }, { "lowered": "l:1016", "source": "s:1517", "status": "whole" }, { "lowered": "l:1017", "source": "s:1518", "status": "whole" }, { "lowered": "l:1018", "source": "s:1519", "status": "whole" }, { "source": "s:1520", "status": "not_materialized" }, { "lowered": "l:1019", "source": "s:1521", "status": "whole" }, { "lowered": "l:1020", "source": "s:1522", "status": "whole" }, { "lowered": "l:1021", "source": "s:1523", "status": "whole" }, { "source": "s:1524", "status": "not_materialized" }, { "lowered": "l:1022", "source": "s:1525", "status": "whole" }, { "lowered": "l:1023", "source": "s:1526", "status": "whole" }, { "lowered": "l:1024", "source": "s:1527", "status": "whole" }, { "lowered": "l:1025", "source": "s:1528", "status": "whole" }, { "lowered": "l:1026", "source": "s:1529", "status": "whole" }, { "lowered": "l:1027", "source": "s:1530", "status": "whole" }, { "lowered": "l:1028", "source": "s:1531", "status": "whole" }, { "lowered": "l:1029", "source": "s:1532", "status": "whole" }, { "lowered": "l:1030", "source": "s:1533", "status": "whole" }, { "lowered": "l:1031", "source": "s:1534", "status": "whole" }, { "lowered": "l:1032", "source": "s:1535", "status": "whole" }, { "lowered": "l:1033", "source": "s:1536", "status": "whole" }, { "lowered": "l:1034", "source": "s:1537", "status": "whole" }, { "source": "s:1538", "status": "not_materialized" }, { "source": "s:1539", "status": "not_materialized" }, { "source": "s:1540", "status": "not_materialized" }, { "lowered": "l:1035", "source": "s:1541", "status": "whole" }, { "lowered": "l:1036", "source": "s:1542", "status": "whole" }, { "lowered": "l:1037", "source": "s:1543", "status": "whole" }, { "source": "s:1544", "status": "not_materialized" }, { "lowered": "l:1038", "source": "s:1545", "status": "whole" }, { "lowered": "l:1039", "source": "s:1546", "status": "whole" }, { "source": "s:1547", "status": "not_materialized" }, { "source": "s:1548", "status": "not_materialized" }, { "source": "s:1549", "status": "not_materialized" }, { "source": "s:1550", "status": "not_materialized" }, { "lowered": "l:1040", "source": "s:1551", "status": "whole" }, { "lowered": "l:1041", "source": "s:1552", "status": "whole" }, { "lowered": "l:1042", "source": "s:1553", "status": "whole" }, { "source": "s:1554", "status": "not_materialized" }, { "lowered": "l:1043", "source": "s:1555", "status": "whole" }, { "lowered": "l:1044", "source": "s:1556", "status": "whole" }, { "lowered": "l:1045", "source": "s:1557", "status": "whole" }, { "source": "s:1558", "status": "not_materialized" }, { "lowered": "l:1046", "source": "s:1559", "status": "whole" }, { "lowered": "l:1047", "source": "s:1560", "status": "whole" }, { "lowered": "l:1048", "source": "s:1561", "status": "whole" }, { "lowered": "l:1049", "source": "s:1562", "status": "whole" }, { "lowered": "l:1050", "source": "s:1563", "status": "whole" }, { "lowered": "l:1051", "source": "s:1564", "status": "whole" }, { "lowered": "l:1052", "source": "s:1565", "status": "whole" }, { "lowered": "l:1053", "source": "s:1566", "status": "whole" }, { "lowered": "l:1054", "source": "s:1567", "status": "whole" }, { "lowered": "l:1055", "source": "s:1568", "status": "whole" }, { "lowered": "l:1056", "source": "s:1569", "status": "whole" }, { "lowered": "l:1057", "source": "s:1570", "status": "whole" } ], "source": { "nodes": [ { "annotation": null, "id": "s:0", "inputs": [ "s:1" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=33.40152674999999))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.40152674999999, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.40152674999999, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1", "inputs": [ "s:2", "s:10", "s:1570" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:0", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 33.40152674999999, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:2", "inputs": [ "s:3", "s:6", "s:7" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:2", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:3", "inputs": [ "s:4", "s:5" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:2", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:4", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:2", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:5", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:2", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:6", "inputs": [ "s:7", "s:9" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:2", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:7", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "fe8947e9c75de0c7a722d4a0a01bc7acb10be2bfac772df3c42693c0f839bc75", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:4", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:8", "inputs": [], "name": null, "op": "Embedding", "parameters": [ { "dtype": "float32", "name": "table", "sha256": "f577080080471c775e844c519cb94f68d8b4b70dedb33c17f246ede2a99770c0", "shape": [ 34, 33 ] } ], "region": "r:5", "scheduling_predecessors": [], "semantics": { "embedding_width": 33, "input_name": "embedding_input", "vocab_size": 34 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:9", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:2", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:10", "inputs": [ "s:11" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3bb9bd83a7c5002d056777dc8133aeefb93900d51f26017a1a809df297d7aa12", "shape": [ 891, 33 ] } ], "region": "r:6", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.40152674999999, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:11", "inputs": [ "s:12", "s:81", "s:277", "s:356", "s:405", "s:467", "s:529", "s:586", "s:644", "s:700", "s:763", "s:823", "s:883", "s:941", "s:976", "s:1063", "s:1138", "s:1212", "s:1286", "s:1321", "s:1340", "s:1381", "s:1415", "s:1449", "s:1483", "s:1517", "s:1551" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:6", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.040175249999999, "upper_unbounded": false } }, "width": 891 }, { "annotation": null, "id": "s:12", "inputs": [ "s:13" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:8", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:13", "inputs": [ "s:14", "s:33" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:7", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:14", "inputs": [ "s:15" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:11", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:15", "inputs": [ "s:16" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:12", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:16", "inputs": [ "s:17", "s:31" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:12", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:17", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:14", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.9956489921875, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:18", "inputs": [ "s:19" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:16", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.35743853290536454, "lower_unbounded": false, "upper": 29.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:19", "inputs": [ "s:20" ], "name": "reciprocal_0_45", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "e1ae2d1ffae9a5e717a562c3a3ed22758427a189561eab446d291d61794a6fca", "shape": [ 45 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "12a7bb89b5494ee4cb4831e9ecb33ff0d5d170c222480f293d1af85996ae08d4", "shape": [ 45, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3a1597c5b00dc1a24568403421c53d3fe42330996ee5f8e7aa3853065db6105b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "d47e14c9cf66b4cb3bca428ef8c914d56ae8c8596823bf1d8a69fcb18f2fefbc", "shape": [ 45, 1 ] } ], "region": "r:19", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 45 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6425614670946355, hi=30.52410519957203))" } ], "claimed_range": { "lower": 0.6425614670946355, "lower_unbounded": false, "upper": 30.52410519957203, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6425614670946355, "lower_unbounded": false, "upper": 30.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:20", "inputs": [ "s:21", "s:22", "s:29" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:20", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:21", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:20", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:22", "inputs": [ "s:23", "s:26", "s:27" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:21", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:23", "inputs": [ "s:24", "s:25" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:21", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:24", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:21", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:25", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:21", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:26", "inputs": [ "s:27", "s:28" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:21", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:27", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "fe8947e9c75de0c7a722d4a0a01bc7acb10be2bfac772df3c42693c0f839bc75", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:23", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:28", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:21", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:29", "inputs": [ "s:30" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:25", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:30", "inputs": [ "s:27" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:26", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:31", "inputs": [ "s:32" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:28", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:32", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:29", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:33", "inputs": [ "s:34" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:31", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:34", "inputs": [ "s:35", "s:63", "s:72" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:31", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:35", "inputs": [ "s:36" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:33", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:36", "inputs": [ "s:37", "s:62" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:32", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:37", "inputs": [ "s:38", "s:41", "s:49" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:34", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:38", "inputs": [ "s:39", "s:40" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:34", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:39", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:34", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:40", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:34", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:41", "inputs": [ "s:42", "s:61" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:34", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:42", "inputs": [ "s:43" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:37", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:43", "inputs": [ "s:44" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:38", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:44", "inputs": [ "s:45", "s:54" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:38", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:45", "inputs": [ "s:46" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:41", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:46", "inputs": [ "s:47" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3, 1 ] } ], "region": "r:42", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.00652651171875, "lower_unbounded": false, "upper": 3.00652651171875, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:47", "inputs": [ "s:48", "s:50", "s:52" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:42", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:48", "inputs": [ "s:49" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:44", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:49", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1d96963de1e65d345a01d2579d659fb9db80a45a36f4e0e85de3c7801d2f0bbd", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:46", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:50", "inputs": [ "s:51" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:48", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:51", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "556ef51bf9d180c5c5da1df13d26129671a25b8df8ee97141ecdb18681bcf06f", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:50", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:52", "inputs": [ "s:53" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:52", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:53", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9c5442b9d0ffed8d2673a1ace282ad01a661c3a14793f768ed10d421763cc9a3", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:54", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:54", "inputs": [ "s:55" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:57", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:55", "inputs": [ "s:56", "s:59", "s:7" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:58", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:56", "inputs": [ "s:57", "s:58" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:58", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:57", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:58", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:58", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:58", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:59", "inputs": [ "s:7", "s:60" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:58", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:60", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:58", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:61", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:34", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:62", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "f8cab608d0db37b3ad632c5fb81de1037b5e40c21163ca60a6859b38d59493e5", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:59", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:63", "inputs": [ "s:64" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:61", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:64", "inputs": [ "s:65", "s:71" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:60", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:65", "inputs": [ "s:66", "s:69", "s:51" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:62", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:66", "inputs": [ "s:67", "s:68" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:62", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:67", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:62", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:68", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:62", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:69", "inputs": [ "s:42", "s:70" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:62", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:70", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:62", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:71", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "f8cab608d0db37b3ad632c5fb81de1037b5e40c21163ca60a6859b38d59493e5", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:63", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:72", "inputs": [ "s:73" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:65", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:73", "inputs": [ "s:74", "s:80" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:64", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:74", "inputs": [ "s:75", "s:78", "s:53" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:66", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:75", "inputs": [ "s:76", "s:77" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:66", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:76", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:66", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:77", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:66", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:78", "inputs": [ "s:42", "s:79" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:66", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:79", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:66", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:80", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "f8cab608d0db37b3ad632c5fb81de1037b5e40c21163ca60a6859b38d59493e5", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:67", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:81", "inputs": [ "s:82" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:69", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:82", "inputs": [ "s:83", "s:89" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:68", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:83", "inputs": [ "s:84" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:72", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:84", "inputs": [ "s:85" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:73", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:85", "inputs": [ "s:86", "s:87" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:73", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:86", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:75", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:87", "inputs": [ "s:88" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9bc792642836b330e7b77ccfcb99d0ea4176f98a05cc12439b285610ead907ae", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:77", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:88", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:78", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:89", "inputs": [ "s:90" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:80", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:90", "inputs": [ "s:91", "s:224", "s:262" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:80", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:91", "inputs": [ "s:92" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:82", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:92", "inputs": [ "s:37", "s:93" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:81", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:93", "inputs": [ "s:94" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc4946eea97aae0d51d873e2649a572ef225febf8c1ba00e86e14e627919a941", "shape": [ 20, 33 ] } ], "region": "r:83", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:94", "inputs": [ "s:95" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:85", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:95", "inputs": [ "s:96" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:86", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:96", "inputs": [ "s:97" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:88", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:97", "inputs": [ "s:98", "s:223" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:87", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:98", "inputs": [ "s:99", "s:222" ], "name": null, "op": "Add", "parameters": [], "region": "r:89", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:99", "inputs": [ "s:100", "s:165" ], "name": null, "op": "Add", "parameters": [], "region": "r:90", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:100", "inputs": [ "s:101" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "region": "r:91", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:101", "inputs": [ "s:102", "s:105", "s:107" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:92", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:102", "inputs": [ "s:103", "s:104" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:92", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:103", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:92", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:104", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:92", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:105", "inputs": [ "s:7", "s:106" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:92", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:106", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:92", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:107", "inputs": [ "s:108", "s:137", "s:162" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:94", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:108", "inputs": [ "s:109", "s:135" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:95", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:109", "inputs": [ "s:110" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:97", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:110", "inputs": [ "s:111" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:98", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:111", "inputs": [ "s:112" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:100", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:112", "inputs": [ "s:113", "s:134" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:99", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:113", "inputs": [ "s:114" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:101", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:114", "inputs": [ "s:115" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:103", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:115", "inputs": [ "s:116", "s:119", "s:121" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:104", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:116", "inputs": [ "s:117", "s:118" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:104", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:117", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:104", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:118", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:104", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:119", "inputs": [ "s:42", "s:120" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:104", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:120", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:104", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:121", "inputs": [ "s:122" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:107", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:122", "inputs": [ "s:123" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "86ace85102b9d8a1c0886e9cf2cbb181c89f8ca60093d8ccbe6cbd12a754ad1b", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5870a33a3d94db2db2aaa92ac9edb125e7803a503405c199042a2f0c6d461a6b", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "41d043a7c0f00b8b39f1365727f09ede61e4b865694b6dc1d4acbbd97029d11b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a4d06da2357859d57b942afb3157f9957beb9485b57a8ae112cd6e8f5e6b3764", "shape": [ 32, 1 ] } ], "region": "r:110", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6633250767579517, hi=5.503341589908715))" } ], "claimed_range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:123", "inputs": [ "s:124", "s:125", "s:132" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:111", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:124", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:111", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:125", "inputs": [ "s:126", "s:129", "s:130" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:112", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:126", "inputs": [ "s:127", "s:128" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:112", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:127", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:112", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:128", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:112", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:129", "inputs": [ "s:130", "s:131" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:112", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:130", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "315553db8f515ee433f197f38119115bcfc62f6f3224c0ade3022f47076efcf5", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:114", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:131", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:112", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:132", "inputs": [ "s:133" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:116", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:133", "inputs": [ "s:130" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:117", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:134", "inputs": [ "s:113" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:118", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420485, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:135", "inputs": [ "s:136" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:119", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:136", "inputs": [ "s:109" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:93", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:137", "inputs": [ "s:138", "s:159" ], "name": null, "op": "Add", "parameters": [], "region": "r:120", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.5118727628515622, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:138", "inputs": [ "s:139", "s:158" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:121", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.0093727628515623, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:139", "inputs": [ "s:140" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "acc2e4c62e9b92c67550621c1d3c42498eab8b87200286819aae131530c4dc41", "shape": [ 3, 4 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e5cd6f63a143d03bf475cd19b883bbdecec4efb62d8303f209b0dde6ae0565cd", "shape": [ 3, 3 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "ba88de7e97f50c001a9d0c4e5c690cd5e4d90c7e5a7b64b96b485a3847228885", "shape": [ 3, 4 ] } ], "region": "r:123", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 3 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.004351007812499974, hi=1.0043510078125))" } ], "claimed_range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:140", "inputs": [ "s:141", "s:152" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:122", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:141", "inputs": [ "s:142" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:126", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:142", "inputs": [ "s:143" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:127", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0478611640624997, "lower_unbounded": false, "upper": 20.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:143", "inputs": [ "s:144", "s:145" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:127", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:144", "inputs": [ "s:8" ], "name": "map_to_table", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "2c34ce1df23b838c5abf2a7f6437cca3d3067ed509ff25f11df6b11b582b51eb", "shape": [ 10 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "88eadde2a89aee8855810a153a5cb169a5e92d43ec5b4f1776e6401b5e163fb5", "shape": [ 10, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e7a2f4b7a886aabf181b94c496dabd00bd3f8da32a01f3da03e6d6868564aed6", "shape": [ 10, 1 ] } ], "region": "r:130", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 10 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-21.0, hi=19.0))" } ], "claimed_range": { "lower": -21.0, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:145", "inputs": [ "s:146" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:133", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:146", "inputs": [ "s:147", "s:150", "s:42" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:134", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:147", "inputs": [ "s:148", "s:149" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:134", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:148", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:134", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:149", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:134", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:150", "inputs": [ "s:42", "s:151" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:134", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:151", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:134", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:152", "inputs": [ "s:153" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:136", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:153", "inputs": [ "s:154" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:137", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:154", "inputs": [ "s:155" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:139", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:155", "inputs": [ "s:156", "s:157" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:138", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:156", "inputs": [ "s:121" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:140", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:157", "inputs": [ "s:156" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:141", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:158", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:142", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:159", "inputs": [ "s:160" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6ad01752d1a3928862bafa2e6bbe6b2df45d614b086e065fdacf50cb7180f6f3", "shape": [ 4, 5 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c91315e209cc3cd1c12fae1264b1597c61d8ddc4fb51cc0941c96fd1be2076b1", "shape": [ 4, 4 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "6e92b4c838f9ab461fc45d72308d5a5360af4497fff417bae914541608979a02", "shape": [ 4, 5 ] } ], "region": "r:144", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 4 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:160", "inputs": [ "s:130", "s:161" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:143", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:161", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "024712320e6f0d6455efd690279f94b44c2b7ac0f5351f626965347a9d3d4199", "shape": [ 4 ], "values": [ 0.5, 0.5, 0.5, 1.0 ] } ], "region": "r:145", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.5, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:162", "inputs": [ "s:163" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:147", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:163", "inputs": [ "s:141", "s:8", "s:164" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:146", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:164", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:148", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:165", "inputs": [ "s:166" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "region": "r:149", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:166", "inputs": [ "s:167", "s:170", "s:172" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:150", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:167", "inputs": [ "s:168", "s:169" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:150", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:168", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:150", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:169", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:150", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:170", "inputs": [ "s:7", "s:171" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:150", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:171", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:150", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:172", "inputs": [ "s:173", "s:201", "s:219" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:152", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:173", "inputs": [ "s:174", "s:199" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:153", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:174", "inputs": [ "s:175" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:155", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:175", "inputs": [ "s:176" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:156", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:176", "inputs": [ "s:177" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:158", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:177", "inputs": [ "s:178", "s:198" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:157", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:178", "inputs": [ "s:179" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:159", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:179", "inputs": [ "s:180" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:161", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:180", "inputs": [ "s:181", "s:184", "s:186" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:162", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:181", "inputs": [ "s:182", "s:183" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:162", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:182", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:162", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:183", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:162", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:184", "inputs": [ "s:7", "s:185" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:162", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:185", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:162", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:186", "inputs": [ "s:187" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:165", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:187", "inputs": [ "s:188" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "86ace85102b9d8a1c0886e9cf2cbb181c89f8ca60093d8ccbe6cbd12a754ad1b", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5870a33a3d94db2db2aaa92ac9edb125e7803a503405c199042a2f0c6d461a6b", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "41d043a7c0f00b8b39f1365727f09ede61e4b865694b6dc1d4acbbd97029d11b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a4d06da2357859d57b942afb3157f9957beb9485b57a8ae112cd6e8f5e6b3764", "shape": [ 32, 1 ] } ], "region": "r:168", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6633250767579517, hi=5.503341589908715))" } ], "claimed_range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:188", "inputs": [ "s:189", "s:190", "s:196" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:169", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0021755039062499]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:189", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:169", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:190", "inputs": [ "s:191", "s:194", "s:42" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:170", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:191", "inputs": [ "s:192", "s:193" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:170", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:192", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:170", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:193", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:170", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:194", "inputs": [ "s:42", "s:195" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:170", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:195", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:170", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:196", "inputs": [ "s:197" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:172", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:197", "inputs": [ "s:42" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:173", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:198", "inputs": [ "s:178" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:174", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420485, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:199", "inputs": [ "s:200" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:175", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:200", "inputs": [ "s:174" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:151", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:201", "inputs": [ "s:202", "s:216" ], "name": null, "op": "Add", "parameters": [], "region": "r:176", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.5118727628515622, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:202", "inputs": [ "s:203", "s:215" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:177", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.0093727628515623, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:203", "inputs": [ "s:204" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "acc2e4c62e9b92c67550621c1d3c42498eab8b87200286819aae131530c4dc41", "shape": [ 3, 4 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e5cd6f63a143d03bf475cd19b883bbdecec4efb62d8303f209b0dde6ae0565cd", "shape": [ 3, 3 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "ba88de7e97f50c001a9d0c4e5c690cd5e4d90c7e5a7b64b96b485a3847228885", "shape": [ 3, 4 ] } ], "region": "r:179", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 3 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.004351007812499974, hi=1.0043510078125))" } ], "claimed_range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:204", "inputs": [ "s:205", "s:209" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:178", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:205", "inputs": [ "s:206" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f22239942c4ac043eb6887cd996054943f2d24a5e56217c21186a366ca97c23d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:182", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:206", "inputs": [ "s:207" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3, 1 ] } ], "region": "r:183", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0522121718749995, "lower_unbounded": false, "upper": 21.008702015624998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:207", "inputs": [ "s:144", "s:146", "s:208" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:183", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:208", "inputs": [ "s:55" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:186", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:209", "inputs": [ "s:210" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:188", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:210", "inputs": [ "s:211" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:189", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:211", "inputs": [ "s:212" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:191", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:212", "inputs": [ "s:213", "s:214" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:190", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:213", "inputs": [ "s:186" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:192", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:214", "inputs": [ "s:213" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:193", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:215", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:194", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:216", "inputs": [ "s:217" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6ad01752d1a3928862bafa2e6bbe6b2df45d614b086e065fdacf50cb7180f6f3", "shape": [ 4, 5 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c91315e209cc3cd1c12fae1264b1597c61d8ddc4fb51cc0941c96fd1be2076b1", "shape": [ 4, 4 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "6e92b4c838f9ab461fc45d72308d5a5360af4497fff417bae914541608979a02", "shape": [ 4, 5 ] } ], "region": "r:196", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 4 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:217", "inputs": [ "s:42", "s:218" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:195", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:218", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "024712320e6f0d6455efd690279f94b44c2b7ac0f5351f626965347a9d3d4199", "shape": [ 4 ], "values": [ 0.5, 0.5, 0.5, 1.0 ] } ], "region": "r:197", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.5, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:219", "inputs": [ "s:220" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:199", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:220", "inputs": [ "s:205", "s:8", "s:221" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:198", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:221", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "04bafa6dc3172d513d574271fa268859c486732a683fc563ffe993cc5a34d411", "shape": [ 33 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:200", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:222", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:201", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:223", "inputs": [ "s:98" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:202", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:224", "inputs": [ "s:225" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:204", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:225", "inputs": [ "s:65", "s:226" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:203", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:226", "inputs": [ "s:227" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "bb8fc959ceb3d086d9702adc3f58f39c225857bc7a0f5c41addc2beb5ee1fefd", "shape": [ 3, 33 ] } ], "region": "r:205", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:227", "inputs": [ "s:228" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9c5219c31cbc274e2d589242ce813d94b925fe67b63640f94eef3e6ca029f3bf", "shape": [ 300 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bd021f3ce5360c520035f4f7e6f53fd3221febaa63178551cb2e108177bade04", "shape": [ 300, 69 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3695fc5961dab4c804313662c5662248a3536224fb1dd93f8fb23304c437ff0c", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "5792a0f30ddb29610b18a2899f6a4ddf6c4c4ff947c643fda05bd813e2a31e4c", "shape": [ 300, 3 ] } ], "region": "r:207", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 300 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:228", "inputs": [ "s:229", "s:230", "s:246" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:208", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 69 }, { "annotation": null, "id": "s:229", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "2851d26fefd673cc1134c966bd403171ea775cea78d9c49d1dd489a2931688a8", "shape": [ 33, 3 ] } ], "region": "r:209", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:230", "inputs": [ "s:231", "s:234", "s:236" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:210", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:231", "inputs": [ "s:232", "s:233" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:210", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:232", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:210", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:233", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:210", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:234", "inputs": [ "s:7", "s:235" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:210", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:235", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:210", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:236", "inputs": [ "s:237", "s:137", "s:162" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:212", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:237", "inputs": [ "s:238", "s:244" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:213", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:238", "inputs": [ "s:239" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:215", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:239", "inputs": [ "s:240" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:216", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:240", "inputs": [ "s:241" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:218", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:241", "inputs": [ "s:242", "s:243" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:217", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:242", "inputs": [ "s:114" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "71426d210d52fa91812d0a39251aa75ded92519c3d746b8ced4e5a02ec97960d", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:219", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.3366749232420485, "lower_unbounded": false, "upper": 0.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:243", "inputs": [ "s:242" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:220", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:244", "inputs": [ "s:245" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:221", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:245", "inputs": [ "s:238" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:211", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:246", "inputs": [ "s:247", "s:250", "s:252" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:222", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:247", "inputs": [ "s:248", "s:249" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:222", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:248", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:222", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:249", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:222", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:250", "inputs": [ "s:7", "s:251" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:222", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:251", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:222", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:252", "inputs": [ "s:253", "s:201", "s:219" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:224", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:253", "inputs": [ "s:254", "s:260" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:225", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:254", "inputs": [ "s:255" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:227", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:255", "inputs": [ "s:256" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:228", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:256", "inputs": [ "s:257" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:230", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:257", "inputs": [ "s:258", "s:259" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:229", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:258", "inputs": [ "s:179" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "71426d210d52fa91812d0a39251aa75ded92519c3d746b8ced4e5a02ec97960d", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:231", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.3366749232420485, "lower_unbounded": false, "upper": 0.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:259", "inputs": [ "s:258" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:232", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:260", "inputs": [ "s:261" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:233", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:261", "inputs": [ "s:254" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:223", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:262", "inputs": [ "s:263" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:235", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:263", "inputs": [ "s:74", "s:264" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:234", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:264", "inputs": [ "s:265" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70ba3fb8f29d3fbfb2412bfe206bc0de79c1b96f3ad7119b49f40c81da58c0a1", "shape": [ 61, 33 ] } ], "region": "r:236", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:265", "inputs": [ "s:266" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:238", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:266", "inputs": [ "s:267" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:239", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:267", "inputs": [ "s:268" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:241", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:268", "inputs": [ "s:269", "s:276" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:240", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 10.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:269", "inputs": [ "s:270", "s:275" ], "name": null, "op": "Add", "parameters": [], "region": "r:242", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:270", "inputs": [ "s:271" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:243", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:271", "inputs": [ "s:272" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:243", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:272", "inputs": [ "s:273" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:244", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:273", "inputs": [ "s:274" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:246", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:274", "inputs": [ "s:101", "s:166" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:247", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "s:275", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:248", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:276", "inputs": [ "s:269" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:249", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 10.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:277", "inputs": [ "s:278" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:251", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:278", "inputs": [ "s:279", "s:285" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:250", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:279", "inputs": [ "s:280" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:254", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:280", "inputs": [ "s:281" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:255", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:281", "inputs": [ "s:282", "s:283" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:255", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:282", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:257", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:283", "inputs": [ "s:284" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ed53dbef9c852a8229e4b386e20627c3a89599c7583ec50379471fb4415bc5b3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:259", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:284", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:260", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:285", "inputs": [ "s:286" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:262", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:286", "inputs": [ "s:287", "s:332", "s:337" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:262", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:287", "inputs": [ "s:288" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:264", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:288", "inputs": [ "s:37", "s:289" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:263", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:289", "inputs": [ "s:290" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc4946eea97aae0d51d873e2649a572ef225febf8c1ba00e86e14e627919a941", "shape": [ 20, 33 ] } ], "region": "r:265", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:290", "inputs": [ "s:291" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:267", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:291", "inputs": [ "s:292" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:268", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:292", "inputs": [ "s:293" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:270", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:293", "inputs": [ "s:294", "s:331" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:269", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 25.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:294", "inputs": [ "s:295", "s:330" ], "name": null, "op": "Add", "parameters": [], "region": "r:271", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 24.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:295", "inputs": [ "s:296", "s:313" ], "name": null, "op": "Add", "parameters": [], "region": "r:272", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:296", "inputs": [ "s:297" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "region": "r:273", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:297", "inputs": [ "s:298", "s:301", "s:303" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:274", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:298", "inputs": [ "s:299", "s:300" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:274", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:299", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:274", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:300", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:274", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:301", "inputs": [ "s:7", "s:302" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:274", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:302", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:274", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:303", "inputs": [ "s:304", "s:137", "s:162" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:276", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:304", "inputs": [ "s:305", "s:311" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:277", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:305", "inputs": [ "s:306" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:279", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:306", "inputs": [ "s:307" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:280", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:307", "inputs": [ "s:308" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:282", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:308", "inputs": [ "s:309", "s:310" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:281", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:309", "inputs": [ "s:114" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e4767380eb5e2fc046bce28b8b2a30c81c733be1a56203cd9499066086617f6c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:283", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:310", "inputs": [ "s:309" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:284", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:311", "inputs": [ "s:312" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:285", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:312", "inputs": [ "s:305" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:275", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:313", "inputs": [ "s:314" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "region": "r:286", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:314", "inputs": [ "s:315", "s:318", "s:320" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:287", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:315", "inputs": [ "s:316", "s:317" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:287", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:316", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:287", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:317", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:287", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:318", "inputs": [ "s:7", "s:319" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:287", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:319", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:287", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:320", "inputs": [ "s:321", "s:201", "s:219" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:289", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:321", "inputs": [ "s:322", "s:328" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:290", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:322", "inputs": [ "s:323" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:292", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:323", "inputs": [ "s:324" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:293", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:324", "inputs": [ "s:325" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:295", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:325", "inputs": [ "s:326", "s:327" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:294", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:326", "inputs": [ "s:179" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e4767380eb5e2fc046bce28b8b2a30c81c733be1a56203cd9499066086617f6c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:296", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:327", "inputs": [ "s:326" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:297", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:328", "inputs": [ "s:329" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:298", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:329", "inputs": [ "s:322" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:288", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:330", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "region": "r:299", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:331", "inputs": [ "s:294" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:300", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 25.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:332", "inputs": [ "s:333" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:302", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:333", "inputs": [ "s:65", "s:334" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:301", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:334", "inputs": [ "s:335" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "bb8fc959ceb3d086d9702adc3f58f39c225857bc7a0f5c41addc2beb5ee1fefd", "shape": [ 3, 33 ] } ], "region": "r:303", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:335", "inputs": [ "s:336" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9c5219c31cbc274e2d589242ce813d94b925fe67b63640f94eef3e6ca029f3bf", "shape": [ 300 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bd021f3ce5360c520035f4f7e6f53fd3221febaa63178551cb2e108177bade04", "shape": [ 300, 69 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3695fc5961dab4c804313662c5662248a3536224fb1dd93f8fb23304c437ff0c", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "5792a0f30ddb29610b18a2899f6a4ddf6c4c4ff947c643fda05bd813e2a31e4c", "shape": [ 300, 3 ] } ], "region": "r:305", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 300 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:336", "inputs": [ "s:229", "s:297", "s:314" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:306", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 69 }, { "annotation": null, "id": "s:337", "inputs": [ "s:338" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:308", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:338", "inputs": [ "s:74", "s:339" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:307", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:339", "inputs": [ "s:340" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70ba3fb8f29d3fbfb2412bfe206bc0de79c1b96f3ad7119b49f40c81da58c0a1", "shape": [ 61, 33 ] } ], "region": "r:309", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:340", "inputs": [ "s:341" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:311", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:341", "inputs": [ "s:342" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:312", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:342", "inputs": [ "s:343" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:314", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:343", "inputs": [ "s:344", "s:355" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:313", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 34.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:344", "inputs": [ "s:345", "s:354" ], "name": null, "op": "Add", "parameters": [], "region": "r:315", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:345", "inputs": [ "s:346" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3, 1 ] } ], "region": "r:316", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:346", "inputs": [ "s:347", "s:350", "s:353" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:316", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:347", "inputs": [ "s:348" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:317", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:348", "inputs": [ "s:349" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:319", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:349", "inputs": [ "s:297", "s:166" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:320", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "s:350", "inputs": [ "s:351" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:321", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:351", "inputs": [ "s:352" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:323", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:352", "inputs": [ "s:101", "s:314" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:324", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "s:353", "inputs": [ "s:273" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:325", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:354", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "region": "r:326", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:355", "inputs": [ "s:344" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:327", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 34.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:356", "inputs": [ "s:357" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:329", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:357", "inputs": [ "s:358", "s:364" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:328", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:358", "inputs": [ "s:359" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:332", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:359", "inputs": [ "s:360" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:333", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:360", "inputs": [ "s:361", "s:362" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:333", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:361", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:335", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:362", "inputs": [ "s:363" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9dfa5cc0a4093ad092e8327b9861cf4f1595c50fa4b3f12efc578b78fe5afdd2", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:337", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:363", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:338", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:364", "inputs": [ "s:365" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:340", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:365", "inputs": [ "s:366", "s:378", "s:383" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:340", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:366", "inputs": [ "s:367" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:342", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:367", "inputs": [ "s:37", "s:368" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:341", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:368", "inputs": [ "s:369" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc4946eea97aae0d51d873e2649a572ef225febf8c1ba00e86e14e627919a941", "shape": [ 20, 33 ] } ], "region": "r:343", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:369", "inputs": [ "s:370" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:345", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:370", "inputs": [ "s:371" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:346", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:371", "inputs": [ "s:372" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:348", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:372", "inputs": [ "s:373", "s:377" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:347", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 25.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:373", "inputs": [ "s:374", "s:330" ], "name": null, "op": "Add", "parameters": [], "region": "r:349", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 24.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:374", "inputs": [ "s:375", "s:376" ], "name": null, "op": "Add", "parameters": [], "region": "r:350", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:375", "inputs": [ "s:230" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "region": "r:351", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:376", "inputs": [ "s:246" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "region": "r:352", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:377", "inputs": [ "s:373" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:353", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 25.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:378", "inputs": [ "s:379" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:355", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:379", "inputs": [ "s:65", "s:380" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:354", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:380", "inputs": [ "s:381" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "bb8fc959ceb3d086d9702adc3f58f39c225857bc7a0f5c41addc2beb5ee1fefd", "shape": [ 3, 33 ] } ], "region": "r:356", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:381", "inputs": [ "s:382" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9c5219c31cbc274e2d589242ce813d94b925fe67b63640f94eef3e6ca029f3bf", "shape": [ 300 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bd021f3ce5360c520035f4f7e6f53fd3221febaa63178551cb2e108177bade04", "shape": [ 300, 69 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "3695fc5961dab4c804313662c5662248a3536224fb1dd93f8fb23304c437ff0c", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "5792a0f30ddb29610b18a2899f6a4ddf6c4c4ff947c643fda05bd813e2a31e4c", "shape": [ 300, 3 ] } ], "region": "r:358", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 300 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:382", "inputs": [ "s:229", "s:101", "s:166" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:359", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 69 }, { "annotation": null, "id": "s:383", "inputs": [ "s:384" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:361", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:384", "inputs": [ "s:74", "s:385" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:360", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:385", "inputs": [ "s:386" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70ba3fb8f29d3fbfb2412bfe206bc0de79c1b96f3ad7119b49f40c81da58c0a1", "shape": [ 61, 33 ] } ], "region": "r:362", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:386", "inputs": [ "s:387" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:364", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:387", "inputs": [ "s:388" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:365", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:388", "inputs": [ "s:389" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:367", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:389", "inputs": [ "s:390", "s:404" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:366", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 52.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:390", "inputs": [ "s:391", "s:354" ], "name": null, "op": "Add", "parameters": [], "region": "r:368", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 51.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:391", "inputs": [ "s:392" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:369", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:392", "inputs": [ "s:393", "s:396", "s:399", "s:400", "s:403" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:369", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:393", "inputs": [ "s:394" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:370", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:394", "inputs": [ "s:395" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:372", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:395", "inputs": [ "s:230", "s:166" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:373", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "s:396", "inputs": [ "s:397" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:374", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:397", "inputs": [ "s:398" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:376", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:398", "inputs": [ "s:297", "s:314" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:377", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "s:399", "inputs": [ "s:348" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:378", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:400", "inputs": [ "s:401" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:379", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:401", "inputs": [ "s:402" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:381", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:402", "inputs": [ "s:101", "s:246" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:382", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "s:403", "inputs": [ "s:351" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:383", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:404", "inputs": [ "s:390" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:384", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 52.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:405", "inputs": [ "s:406" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:386", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:406", "inputs": [ "s:407", "s:413" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:385", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:407", "inputs": [ "s:408" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:389", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:408", "inputs": [ "s:409" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:390", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:409", "inputs": [ "s:410", "s:411" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:390", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:410", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:392", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:411", "inputs": [ "s:412" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25eab693bdf7c6ad20a03812dd32b8a7ba27dab587478a5dc1c8b657174ae97d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:394", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:412", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:395", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:413", "inputs": [ "s:414" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:397", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:414", "inputs": [ "s:415", "s:425", "s:447" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:397", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:415", "inputs": [ "s:416" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:399", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:416", "inputs": [ "s:37", "s:417" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:398", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:417", "inputs": [ "s:418" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc4946eea97aae0d51d873e2649a572ef225febf8c1ba00e86e14e627919a941", "shape": [ 20, 33 ] } ], "region": "r:400", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:418", "inputs": [ "s:419" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:402", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:419", "inputs": [ "s:420" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:403", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:420", "inputs": [ "s:421" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:405", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:421", "inputs": [ "s:422", "s:424" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:404", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 7.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:422", "inputs": [ "s:423", "s:330" ], "name": null, "op": "Add", "parameters": [], "region": "r:406", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:423", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:407", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:424", "inputs": [ "s:422" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:408", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 7.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:425", "inputs": [ "s:426" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:410", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:426", "inputs": [ "s:65", "s:427" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:409", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:427", "inputs": [ "s:428" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "197d48eb99e44305d42edd2e5e9135f2198ab6b71386347e102ea15147a41a47", "shape": [ 20, 33 ] } ], "region": "r:411", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:428", "inputs": [ "s:429" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:413", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:429", "inputs": [ "s:430" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:414", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:430", "inputs": [ "s:431" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:416", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:431", "inputs": [ "s:432", "s:446" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:415", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.9550000000000001, "lower_unbounded": false, "upper": 20.045, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:432", "inputs": [ "s:433" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "80c8a717ccd70c8809eb78e6a9591c003e11c721fe0ccaf62fd592abda1a5593", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:417", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.9550000000000001, "lower_unbounded": false, "upper": 19.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:433", "inputs": [ "s:434", "s:444" ], "name": null, "op": "Add", "parameters": [], "region": "r:419", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:434", "inputs": [ "s:435", "s:441" ], "name": null, "op": "Add", "parameters": [], "region": "r:421", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:435", "inputs": [ "s:436" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "region": "r:423", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:436", "inputs": [ "s:437", "s:439", "s:440" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:422", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:437", "inputs": [ "s:438" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:425", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:438", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "region": "r:426", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:439", "inputs": [ "s:101" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "region": "r:427", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:440", "inputs": [ "s:166" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "region": "r:428", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:441", "inputs": [ "s:442" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:429", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:442", "inputs": [ "s:443" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "region": "r:431", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:443", "inputs": [ "s:437", "s:440", "s:439" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:430", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:444", "inputs": [ "s:445" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:432", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:445", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:433", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:446", "inputs": [ "s:432" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:434", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.955, "lower_unbounded": false, "upper": 20.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:447", "inputs": [ "s:448" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:436", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:448", "inputs": [ "s:74", "s:449" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:435", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:449", "inputs": [ "s:450" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70ba3fb8f29d3fbfb2412bfe206bc0de79c1b96f3ad7119b49f40c81da58c0a1", "shape": [ 61, 33 ] } ], "region": "r:437", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:450", "inputs": [ "s:451" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:439", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:451", "inputs": [ "s:452" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:440", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:452", "inputs": [ "s:453" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:442", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:453", "inputs": [ "s:454", "s:466" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:441", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 52.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:454", "inputs": [ "s:455", "s:354" ], "name": null, "op": "Add", "parameters": [], "region": "r:443", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 51.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:455", "inputs": [ "s:456" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:444", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:456", "inputs": [ "s:457", "s:460", "s:461", "s:464", "s:465" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:444", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:457", "inputs": [ "s:458" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:445", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:458", "inputs": [ "s:459" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:447", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:459", "inputs": [ "s:230", "s:314" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:448", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "s:460", "inputs": [ "s:394" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:449", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:461", "inputs": [ "s:462" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:450", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:462", "inputs": [ "s:463" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:452", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:463", "inputs": [ "s:297", "s:246" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:453", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "s:464", "inputs": [ "s:397" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:454", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:465", "inputs": [ "s:401" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:455", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:466", "inputs": [ "s:454" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:456", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 52.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:467", "inputs": [ "s:468" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:458", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:468", "inputs": [ "s:469", "s:475" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:457", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:469", "inputs": [ "s:470" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:461", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:470", "inputs": [ "s:471" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:462", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:471", "inputs": [ "s:472", "s:473" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:462", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:472", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "db8d1a556ec9527f63930661298f4b8607cfcecb3c364235751acfef1ec8ed1a", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:464", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:473", "inputs": [ "s:474" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "64f403ba374325ca540cb27c0375a0ea1dc20ecaeb654fcbe8bea2b756826dab", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:466", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:474", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:467", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:475", "inputs": [ "s:476" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:469", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:476", "inputs": [ "s:477", "s:489", "s:513" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:469", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:477", "inputs": [ "s:478" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:471", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:478", "inputs": [ "s:37", "s:479" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:470", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:479", "inputs": [ "s:480" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "a173a5b48faee60f7589a0e6b946c552837942a76d3222cd2d9e81c4f3b5c71c", "shape": [ 20, 33 ] } ], "region": "r:472", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:480", "inputs": [ "s:481" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:474", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:481", "inputs": [ "s:482" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:475", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:482", "inputs": [ "s:483" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:477", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:483", "inputs": [ "s:484", "s:488" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:476", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 7.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:484", "inputs": [ "s:423", "s:485" ], "name": null, "op": "Add", "parameters": [], "region": "r:478", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:485", "inputs": [ "s:486" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "region": "r:479", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:486", "inputs": [ "s:487", "s:487", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:480", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:487", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:480", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:488", "inputs": [ "s:484" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:481", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 7.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:489", "inputs": [ "s:490" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:483", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:490", "inputs": [ "s:65", "s:491" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:482", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:491", "inputs": [ "s:492" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "197d48eb99e44305d42edd2e5e9135f2198ab6b71386347e102ea15147a41a47", "shape": [ 20, 33 ] } ], "region": "r:484", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:492", "inputs": [ "s:493" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:486", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:493", "inputs": [ "s:494" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:487", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:494", "inputs": [ "s:495" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:489", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:495", "inputs": [ "s:496", "s:512" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:488", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -5.045, "lower_unbounded": false, "upper": 20.045, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:496", "inputs": [ "s:497" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "80c8a717ccd70c8809eb78e6a9591c003e11c721fe0ccaf62fd592abda1a5593", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:490", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -5.045, "lower_unbounded": false, "upper": 19.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:497", "inputs": [ "s:498", "s:510" ], "name": null, "op": "Add", "parameters": [], "region": "r:492", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -15.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:498", "inputs": [ "s:499", "s:507" ], "name": null, "op": "Add", "parameters": [], "region": "r:494", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:499", "inputs": [ "s:500" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "region": "r:496", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:500", "inputs": [ "s:501", "s:505", "s:506" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:495", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:501", "inputs": [ "s:502" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:498", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:502", "inputs": [ "s:503" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "region": "r:499", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:503", "inputs": [ "s:504", "s:504", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:500", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:504", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:500", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:505", "inputs": [ "s:297" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "region": "r:501", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:506", "inputs": [ "s:314" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "region": "r:502", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:507", "inputs": [ "s:508" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:503", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:508", "inputs": [ "s:509" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "region": "r:505", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:509", "inputs": [ "s:501", "s:506", "s:505" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:504", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:510", "inputs": [ "s:511" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:506", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -6.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:511", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "region": "r:507", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:512", "inputs": [ "s:496" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:508", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.045, "lower_unbounded": false, "upper": 20.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:513", "inputs": [ "s:514" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:510", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:514", "inputs": [ "s:74", "s:515" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:509", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:515", "inputs": [ "s:516" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70ba3fb8f29d3fbfb2412bfe206bc0de79c1b96f3ad7119b49f40c81da58c0a1", "shape": [ 61, 33 ] } ], "region": "r:511", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:516", "inputs": [ "s:517" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:513", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:517", "inputs": [ "s:518" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:514", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:518", "inputs": [ "s:519" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:516", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:519", "inputs": [ "s:520", "s:528" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:515", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 34.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:520", "inputs": [ "s:521", "s:354" ], "name": null, "op": "Add", "parameters": [], "region": "r:517", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:521", "inputs": [ "s:522" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3, 1 ] } ], "region": "r:518", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:522", "inputs": [ "s:523", "s:526", "s:527" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:518", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:523", "inputs": [ "s:524" ], "name": "product_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "22b6f43bd8d27738d3213f29e96b62d01d9d6c0ab4f9732aaae803186f51eab7", "shape": [ 2, 1 ] } ], "region": "r:519", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:524", "inputs": [ "s:525" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "7dffd23b532200283ea0e9c51bcb14241b51b1d63cb88607237f6aa02279d653", "shape": [ 100, 66 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "930247b5fd91aedf5b2b385752ccc1528049604cb3fe0ae6f86982d74bf75d7e", "shape": [ 100, 2 ] } ], "region": "r:521", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:525", "inputs": [ "s:230", "s:246" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:522", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 66 }, { "annotation": null, "id": "s:526", "inputs": [ "s:458" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:523", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:527", "inputs": [ "s:462" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:524", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:528", "inputs": [ "s:520" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:525", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 34.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:529", "inputs": [ "s:530" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:527", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:530", "inputs": [ "s:531", "s:537" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:526", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:531", "inputs": [ "s:532" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:530", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:532", "inputs": [ "s:533" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:531", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:533", "inputs": [ "s:534", "s:535" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:531", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:534", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b8bb7a27e55e02d67b43582292d3134e5715248dc3f4a54dac7e1c71519551b5", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:533", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:535", "inputs": [ "s:536" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df1b487fa89261e6181b4f1197a2248f295f75619fa52d6c77c2a438a9f5862d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:535", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:536", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:536", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:537", "inputs": [ "s:538" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:538", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:538", "inputs": [ "s:539", "s:551", "s:574" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:538", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:539", "inputs": [ "s:540" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:540", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:540", "inputs": [ "s:37", "s:541" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:539", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:541", "inputs": [ "s:542" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "a173a5b48faee60f7589a0e6b946c552837942a76d3222cd2d9e81c4f3b5c71c", "shape": [ 20, 33 ] } ], "region": "r:541", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:542", "inputs": [ "s:543" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:543", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:543", "inputs": [ "s:544" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:544", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:544", "inputs": [ "s:545" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:546", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:545", "inputs": [ "s:546", "s:550" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:545", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 25.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:546", "inputs": [ "s:374", "s:547" ], "name": null, "op": "Add", "parameters": [], "region": "r:547", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 24.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:547", "inputs": [ "s:548" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "region": "r:548", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:548", "inputs": [ "s:549", "s:549", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:549", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:549", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:549", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:550", "inputs": [ "s:546" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:550", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 25.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:551", "inputs": [ "s:552" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:552", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:552", "inputs": [ "s:65", "s:553" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:551", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:553", "inputs": [ "s:554" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "197d48eb99e44305d42edd2e5e9135f2198ab6b71386347e102ea15147a41a47", "shape": [ 20, 33 ] } ], "region": "r:553", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:554", "inputs": [ "s:555" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:555", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:555", "inputs": [ "s:556" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:556", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:556", "inputs": [ "s:557" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:558", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:557", "inputs": [ "s:558", "s:573" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:557", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -5.045, "lower_unbounded": false, "upper": 20.045, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:558", "inputs": [ "s:559" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "80c8a717ccd70c8809eb78e6a9591c003e11c721fe0ccaf62fd592abda1a5593", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:559", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -5.045, "lower_unbounded": false, "upper": 19.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:559", "inputs": [ "s:560", "s:572" ], "name": null, "op": "Add", "parameters": [], "region": "r:561", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -15.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:560", "inputs": [ "s:561", "s:569" ], "name": null, "op": "Add", "parameters": [], "region": "r:563", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:561", "inputs": [ "s:562" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "region": "r:565", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:562", "inputs": [ "s:563", "s:567", "s:568" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:564", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:563", "inputs": [ "s:564" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:567", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:564", "inputs": [ "s:565" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "region": "r:568", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:565", "inputs": [ "s:566", "s:566", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:569", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:566", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:569", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:567", "inputs": [ "s:230" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "region": "r:570", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:568", "inputs": [ "s:246" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3859903431e10a65fb312b314095376a6804c204f6146b8e722dbf3b1c3da0b9", "shape": [ 33, 1 ] } ], "region": "r:571", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" }, { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:569", "inputs": [ "s:570" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:572", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:570", "inputs": [ "s:571" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "region": "r:574", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:571", "inputs": [ "s:563", "s:568", "s:567" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:573", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:572", "inputs": [ "s:511" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:575", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -6.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:573", "inputs": [ "s:558" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:576", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.045, "lower_unbounded": false, "upper": 20.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:574", "inputs": [ "s:575" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:578", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:575", "inputs": [ "s:74", "s:576" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:577", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:576", "inputs": [ "s:577" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70ba3fb8f29d3fbfb2412bfe206bc0de79c1b96f3ad7119b49f40c81da58c0a1", "shape": [ 61, 33 ] } ], "region": "r:579", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:577", "inputs": [ "s:578" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:581", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:578", "inputs": [ "s:579" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:582", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:579", "inputs": [ "s:580" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:584", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:580", "inputs": [ "s:581", "s:585" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:583", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 16.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:581", "inputs": [ "s:582", "s:354" ], "name": null, "op": "Add", "parameters": [], "region": "r:585", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 15.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:582", "inputs": [ "s:583" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:586", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": true, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:583", "inputs": [ "s:584" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:586", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:584", "inputs": [ "s:524" ], "name": "product_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2, 1 ] } ], "region": "r:587", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:585", "inputs": [ "s:581" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:588", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 16.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:586", "inputs": [ "s:587" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:590", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:587", "inputs": [ "s:588", "s:594" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:589", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:588", "inputs": [ "s:589" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:593", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:589", "inputs": [ "s:590" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:594", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:590", "inputs": [ "s:591", "s:592" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:594", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:591", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "fcc789a616d690b13c0ce2bb32111dec233ff36515f19e597b74cfe3b5615bcd", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:596", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:592", "inputs": [ "s:593" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ee1c1e99aec431b3878503f437c2d6ed686bab105f28857153577a62843be75e", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:598", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:593", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:599", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:594", "inputs": [ "s:595" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:601", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:595", "inputs": [ "s:596", "s:608", "s:632" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:601", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:596", "inputs": [ "s:597" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:603", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:597", "inputs": [ "s:37", "s:598" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:602", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:598", "inputs": [ "s:599" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "a173a5b48faee60f7589a0e6b946c552837942a76d3222cd2d9e81c4f3b5c71c", "shape": [ 20, 33 ] } ], "region": "r:604", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:599", "inputs": [ "s:600" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:606", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:600", "inputs": [ "s:601" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:607", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:601", "inputs": [ "s:602" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:609", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:602", "inputs": [ "s:603", "s:607" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:608", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 25.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:603", "inputs": [ "s:295", "s:604" ], "name": null, "op": "Add", "parameters": [], "region": "r:610", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 24.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:604", "inputs": [ "s:605" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "region": "r:611", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:605", "inputs": [ "s:606", "s:606", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:612", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:606", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:612", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:607", "inputs": [ "s:603" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:613", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 25.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:608", "inputs": [ "s:609" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:615", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:609", "inputs": [ "s:65", "s:610" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:614", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:610", "inputs": [ "s:611" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "a173a5b48faee60f7589a0e6b946c552837942a76d3222cd2d9e81c4f3b5c71c", "shape": [ 20, 33 ] } ], "region": "r:616", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:611", "inputs": [ "s:612" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:618", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:612", "inputs": [ "s:613" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:619", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:613", "inputs": [ "s:614" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:621", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:614", "inputs": [ "s:615", "s:631" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:620", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -5.045, "lower_unbounded": false, "upper": 20.045, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:615", "inputs": [ "s:616" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "80c8a717ccd70c8809eb78e6a9591c003e11c721fe0ccaf62fd592abda1a5593", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:622", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -5.045, "lower_unbounded": false, "upper": 19.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:616", "inputs": [ "s:617", "s:627" ], "name": null, "op": "Add", "parameters": [], "region": "r:624", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -15.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:617", "inputs": [ "s:618", "s:624" ], "name": null, "op": "Add", "parameters": [], "region": "r:626", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:618", "inputs": [ "s:619" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "region": "r:628", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:619", "inputs": [ "s:620", "s:567", "s:568" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:627", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:620", "inputs": [ "s:621" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:630", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:621", "inputs": [ "s:622" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "region": "r:631", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:622", "inputs": [ "s:623", "s:623", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:632", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:623", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:632", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:624", "inputs": [ "s:625" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:633", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:625", "inputs": [ "s:626" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "region": "r:635", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:626", "inputs": [ "s:620", "s:568", "s:567" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:634", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:627", "inputs": [ "s:628" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:636", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -6.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:628", "inputs": [ "s:629" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "region": "r:637", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:629", "inputs": [ "s:630", "s:630", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:638", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:630", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:638", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:631", "inputs": [ "s:615" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:639", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.045, "lower_unbounded": false, "upper": 20.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:632", "inputs": [ "s:633" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:641", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:633", "inputs": [ "s:74", "s:634" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:640", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:634", "inputs": [ "s:635" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "67bbf7e8bef0cb24fa16b53fe5abf8ad64ff6177c3a9c829199acc4d3861b5ce", "shape": [ 61, 33 ] } ], "region": "r:642", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:635", "inputs": [ "s:636" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:644", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:636", "inputs": [ "s:637" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:645", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:637", "inputs": [ "s:638" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:647", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:638", "inputs": [ "s:639", "s:643" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:646", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 16.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:639", "inputs": [ "s:582", "s:640" ], "name": null, "op": "Add", "parameters": [], "region": "r:648", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 15.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:640", "inputs": [ "s:641" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "region": "r:649", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:641", "inputs": [ "s:642", "s:642", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:650", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:642", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:650", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:643", "inputs": [ "s:639" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:651", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 16.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:644", "inputs": [ "s:645" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:653", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:645", "inputs": [ "s:646", "s:652" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:652", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:646", "inputs": [ "s:647" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:656", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:647", "inputs": [ "s:648" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:657", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:648", "inputs": [ "s:649", "s:650" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:657", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:649", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "bd2babfb9c1d66c869f90c4f5d274aa44df07a3fabd7bafefbc4f8da66c7ffad", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:659", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:650", "inputs": [ "s:651" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "e1eb8965d53f3eec81b3ef488acc3641be29be59d2f6e900aba04da322a11aa7", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:661", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:651", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:662", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:652", "inputs": [ "s:653" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:664", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:653", "inputs": [ "s:654", "s:664", "s:688" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:664", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:654", "inputs": [ "s:655" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:666", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:655", "inputs": [ "s:37", "s:656" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:665", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:656", "inputs": [ "s:657" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "a173a5b48faee60f7589a0e6b946c552837942a76d3222cd2d9e81c4f3b5c71c", "shape": [ 20, 33 ] } ], "region": "r:667", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:657", "inputs": [ "s:658" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:669", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:658", "inputs": [ "s:659" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:670", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:659", "inputs": [ "s:660" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:672", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:660", "inputs": [ "s:661", "s:663" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:671", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:661", "inputs": [ "s:99", "s:662" ], "name": null, "op": "Add", "parameters": [], "region": "r:673", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:662", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:674", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:663", "inputs": [ "s:661" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:675", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:664", "inputs": [ "s:665" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:677", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:665", "inputs": [ "s:65", "s:666" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:676", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:666", "inputs": [ "s:667" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "a173a5b48faee60f7589a0e6b946c552837942a76d3222cd2d9e81c4f3b5c71c", "shape": [ 20, 33 ] } ], "region": "r:678", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:667", "inputs": [ "s:668" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:680", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:668", "inputs": [ "s:669" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:681", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:669", "inputs": [ "s:670" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:683", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:670", "inputs": [ "s:671", "s:687" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:682", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -5.045, "lower_unbounded": false, "upper": 20.045, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:671", "inputs": [ "s:672" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "80c8a717ccd70c8809eb78e6a9591c003e11c721fe0ccaf62fd592abda1a5593", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:684", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -5.045, "lower_unbounded": false, "upper": 19.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:672", "inputs": [ "s:673", "s:683" ], "name": null, "op": "Add", "parameters": [], "region": "r:686", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -15.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:673", "inputs": [ "s:674", "s:680" ], "name": null, "op": "Add", "parameters": [], "region": "r:688", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:674", "inputs": [ "s:675" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "region": "r:690", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:675", "inputs": [ "s:676", "s:505", "s:506" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:689", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:676", "inputs": [ "s:677" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:692", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:677", "inputs": [ "s:678" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "region": "r:693", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:678", "inputs": [ "s:679", "s:679", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "4a219c06af2467623290b375e580c952ecdae33edf14aa594ac393d27e76b67e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:694", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:679", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:694", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:680", "inputs": [ "s:681" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:695", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:681", "inputs": [ "s:682" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "region": "r:697", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:682", "inputs": [ "s:676", "s:506", "s:505" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:696", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:683", "inputs": [ "s:684" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:698", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -6.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:684", "inputs": [ "s:685" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "region": "r:699", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:685", "inputs": [ "s:686", "s:686", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:700", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:686", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:700", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:687", "inputs": [ "s:671" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:701", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.045, "lower_unbounded": false, "upper": 20.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:688", "inputs": [ "s:689" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:703", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:689", "inputs": [ "s:74", "s:690" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:702", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:690", "inputs": [ "s:691" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "67bbf7e8bef0cb24fa16b53fe5abf8ad64ff6177c3a9c829199acc4d3861b5ce", "shape": [ 61, 33 ] } ], "region": "r:704", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:691", "inputs": [ "s:692" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:706", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:692", "inputs": [ "s:693" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:707", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:693", "inputs": [ "s:694" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:709", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:694", "inputs": [ "s:695", "s:699" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:708", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 34.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:695", "inputs": [ "s:521", "s:696" ], "name": null, "op": "Add", "parameters": [], "region": "r:710", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:696", "inputs": [ "s:697" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "region": "r:711", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:697", "inputs": [ "s:698", "s:698", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:712", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:698", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:712", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:699", "inputs": [ "s:695" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:713", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 34.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:700", "inputs": [ "s:701" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:715", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:701", "inputs": [ "s:702", "s:708" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:714", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:702", "inputs": [ "s:703" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:718", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:703", "inputs": [ "s:704" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:719", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:704", "inputs": [ "s:705", "s:706" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:719", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:705", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b523f26d8d2fa76eb7653ecf593398e72ab630efde060b02aa9e7744b3faa6e0", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:721", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:706", "inputs": [ "s:707" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "46e48c3d02450eeca0c725e363868a61942d805b81019328f5624cc7be3e802b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:723", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:707", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:724", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:708", "inputs": [ "s:709" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:726", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:709", "inputs": [ "s:710", "s:729", "s:751" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:726", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:710", "inputs": [ "s:711" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:728", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:711", "inputs": [ "s:37", "s:712" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:727", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:712", "inputs": [ "s:713" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc0a03a8e9c3e933278bf7f2667ac150ebffc7b84c9d541ff71e6243a22d70ca", "shape": [ 7, 33 ] } ], "region": "r:729", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:713", "inputs": [ "s:714" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "3addfb141cd7c9c4c6543a82191a3707ac29c7a041217782e61d4d91c691aee8", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "0a2ff5f661b6232dedd8c1bb0416601df78831ad504c44bd61acd771d4cbdd95", "shape": [ 7, 7 ] } ], "region": "r:731", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:714", "inputs": [ "s:715" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6f3a1a9d59ae648b4f66e10802905ef54f901e2409b9e01e653a05fd7c4ff786", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "68e014055f97f84bdfabe73267c9ca5a45445840d62edf66ebb0ad398c1d088d", "shape": [ 7, 7 ] } ], "region": "r:732", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:715", "inputs": [ "s:716" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ecd290af9b878d500003aada26fc578ffe18fb71ca439df97b2c5da3577bb637", "shape": [ 28 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ac723062162c7c0744a28c7316bd9750d4304fa46e93681ca5ee8499e8dd7cc9", "shape": [ 28, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "adc7fd999de1c34f9ca31a3d93b5f0a159b3c9ff4d6d8ec10bcf843b21f3f38b", "shape": [ 7 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a87893910c503ca82fca06d9c0032304853a7ad9a6b99559819b7f43eb87d121", "shape": [ 28, 7 ] } ], "region": "r:734", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 28 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:716", "inputs": [ "s:717", "s:728" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:733", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 2.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:717", "inputs": [ "s:718", "s:719" ], "name": null, "op": "Add", "parameters": [], "region": "r:735", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:718", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:736", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:719", "inputs": [ "s:720" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:738", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:720", "inputs": [ "s:721" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:739", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:721", "inputs": [ "s:722" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:742", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:722", "inputs": [ "s:723" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:743", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:723", "inputs": [ "s:724", "s:725" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:743", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:724", "inputs": [ "s:718" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:745", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.9956489921875, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:725", "inputs": [ "s:726" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:747", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:726", "inputs": [ "s:727", "s:727", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:748", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:727", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:748", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:728", "inputs": [ "s:717" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:749", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 2.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:729", "inputs": [ "s:730" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:751", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:730", "inputs": [ "s:65", "s:731" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:750", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:731", "inputs": [ "s:732" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "a173a5b48faee60f7589a0e6b946c552837942a76d3222cd2d9e81c4f3b5c71c", "shape": [ 20, 33 ] } ], "region": "r:752", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:732", "inputs": [ "s:733" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "5b6fb58e61fa475939767d68a446f97f1bff02c0e5935a3ea8bb51e6515783d8", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8d9e8bbeaaf43c3880278beba818fd22ce4e6eb4ef221973f5168af959da1165", "shape": [ 20, 20 ] } ], "region": "r:754", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:733", "inputs": [ "s:734" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "822d06342bcaebfe6cffcbf978ac916b9b7e7fc9e7b788b5e9d74af71c2e5eef", "shape": [ 20 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d194201c65fca2fd877f72d989ad57074ba0716a3d7b839ba28e2753f4a113b5", "shape": [ 20, 20 ] } ], "region": "r:755", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:734", "inputs": [ "s:735" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b4514a6058c531465c7c2154a499e76512d774c04e327b455a41d8954f0f323e", "shape": [ 80 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2634fbe4c46bd8f90ebeb2ee0439d6b94584f1056868a8d55481e074439ca576", "shape": [ 80, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1aca3c40e17aad0c7ebd217df46b5342e629b9f994a212214c792fd54fef8d66", "shape": [ 20 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "522763f344d34bdf33db02cffd131e3998cab9be401ff93a365b40c65d7a2a88", "shape": [ 80, 20 ] } ], "region": "r:757", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 80 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 20 }, { "annotation": null, "id": "s:735", "inputs": [ "s:736", "s:750" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:756", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.9550000000000001, "lower_unbounded": false, "upper": 20.045, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:736", "inputs": [ "s:737" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "80c8a717ccd70c8809eb78e6a9591c003e11c721fe0ccaf62fd592abda1a5593", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:758", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.9550000000000001, "lower_unbounded": false, "upper": 19.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:737", "inputs": [ "s:738", "s:748" ], "name": null, "op": "Add", "parameters": [], "region": "r:760", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:738", "inputs": [ "s:739", "s:745" ], "name": null, "op": "Add", "parameters": [], "region": "r:762", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 9.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:739", "inputs": [ "s:740" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "region": "r:764", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:740", "inputs": [ "s:741", "s:439", "s:440" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:763", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:741", "inputs": [ "s:742" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:766", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:742", "inputs": [ "s:743" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "region": "r:767", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:743", "inputs": [ "s:744", "s:744", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:768", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:744", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:768", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:745", "inputs": [ "s:746" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:769", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.045, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:746", "inputs": [ "s:747" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9de9c2dee122d98164466018a03abb03603fed0628bea11b93dc950792f9113e", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "fbdd7f2e63eca5cce856dfaea70fe152bfb312d7f539ec34ac81d4a153bc0d03", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "395be1edcb9c3701df4479ac0ea17096bf1628a10df12335a860237dc6ef4096", "shape": [ 2, 3 ] } ], "region": "r:771", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=9.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:747", "inputs": [ "s:741", "s:440", "s:439" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:770", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:748", "inputs": [ "s:749" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:772", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:749", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:773", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:750", "inputs": [ "s:736" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:774", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.955, "lower_unbounded": false, "upper": 20.045, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:751", "inputs": [ "s:752" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:776", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:752", "inputs": [ "s:74", "s:753" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:775", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:753", "inputs": [ "s:754" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "67bbf7e8bef0cb24fa16b53fe5abf8ad64ff6177c3a9c829199acc4d3861b5ce", "shape": [ 61, 33 ] } ], "region": "r:777", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:754", "inputs": [ "s:755" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:779", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:755", "inputs": [ "s:756" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:780", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:756", "inputs": [ "s:757" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:782", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:757", "inputs": [ "s:758", "s:762" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:781", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 52.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:758", "inputs": [ "s:455", "s:759" ], "name": null, "op": "Add", "parameters": [], "region": "r:783", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 51.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:759", "inputs": [ "s:760" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "region": "r:784", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:760", "inputs": [ "s:761", "s:761", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:785", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:761", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:785", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:762", "inputs": [ "s:758" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:786", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 52.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:763", "inputs": [ "s:764" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:788", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:764", "inputs": [ "s:765", "s:771" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:787", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:765", "inputs": [ "s:766" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:791", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:766", "inputs": [ "s:767" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:792", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:767", "inputs": [ "s:768", "s:769" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:792", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:768", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "22886717dc687b1348f7927d52b5cc1dc2d3824d3a43c602bf3a48e24296a1be", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:794", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:769", "inputs": [ "s:770" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "41b28832e878ed27557031635fd40c492bfee66e307f10662e84521274fee90c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:796", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:770", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:797", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:771", "inputs": [ "s:772" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:799", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:772", "inputs": [ "s:773", "s:792", "s:811" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:799", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:773", "inputs": [ "s:774" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:801", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:774", "inputs": [ "s:37", "s:775" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:800", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:775", "inputs": [ "s:776" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc0a03a8e9c3e933278bf7f2667ac150ebffc7b84c9d541ff71e6243a22d70ca", "shape": [ 7, 33 ] } ], "region": "r:802", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:776", "inputs": [ "s:777" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "3addfb141cd7c9c4c6543a82191a3707ac29c7a041217782e61d4d91c691aee8", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "0a2ff5f661b6232dedd8c1bb0416601df78831ad504c44bd61acd771d4cbdd95", "shape": [ 7, 7 ] } ], "region": "r:804", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:777", "inputs": [ "s:778" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6f3a1a9d59ae648b4f66e10802905ef54f901e2409b9e01e653a05fd7c4ff786", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "68e014055f97f84bdfabe73267c9ca5a45445840d62edf66ebb0ad398c1d088d", "shape": [ 7, 7 ] } ], "region": "r:805", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:778", "inputs": [ "s:779" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ecd290af9b878d500003aada26fc578ffe18fb71ca439df97b2c5da3577bb637", "shape": [ 28 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ac723062162c7c0744a28c7316bd9750d4304fa46e93681ca5ee8499e8dd7cc9", "shape": [ 28, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "adc7fd999de1c34f9ca31a3d93b5f0a159b3c9ff4d6d8ec10bcf843b21f3f38b", "shape": [ 7 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a87893910c503ca82fca06d9c0032304853a7ad9a6b99559819b7f43eb87d121", "shape": [ 28, 7 ] } ], "region": "r:807", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 28 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:779", "inputs": [ "s:780", "s:791" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:806", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:780", "inputs": [ "s:781", "s:782" ], "name": null, "op": "Add", "parameters": [], "region": "r:808", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:781", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:809", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:782", "inputs": [ "s:783" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:811", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:783", "inputs": [ "s:784" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:812", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:784", "inputs": [ "s:785" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:815", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:785", "inputs": [ "s:786" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:816", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:786", "inputs": [ "s:787", "s:788" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:816", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:787", "inputs": [ "s:781" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:818", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:788", "inputs": [ "s:789" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:820", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:789", "inputs": [ "s:790", "s:790", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:821", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:790", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:821", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:791", "inputs": [ "s:780" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:822", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:792", "inputs": [ "s:793" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:824", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:793", "inputs": [ "s:65", "s:794" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:823", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:794", "inputs": [ "s:795" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc0a03a8e9c3e933278bf7f2667ac150ebffc7b84c9d541ff71e6243a22d70ca", "shape": [ 7, 33 ] } ], "region": "r:825", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:795", "inputs": [ "s:796" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "3addfb141cd7c9c4c6543a82191a3707ac29c7a041217782e61d4d91c691aee8", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "0a2ff5f661b6232dedd8c1bb0416601df78831ad504c44bd61acd771d4cbdd95", "shape": [ 7, 7 ] } ], "region": "r:827", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:796", "inputs": [ "s:797" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6f3a1a9d59ae648b4f66e10802905ef54f901e2409b9e01e653a05fd7c4ff786", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "68e014055f97f84bdfabe73267c9ca5a45445840d62edf66ebb0ad398c1d088d", "shape": [ 7, 7 ] } ], "region": "r:828", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:797", "inputs": [ "s:798" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ecd290af9b878d500003aada26fc578ffe18fb71ca439df97b2c5da3577bb637", "shape": [ 28 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ac723062162c7c0744a28c7316bd9750d4304fa46e93681ca5ee8499e8dd7cc9", "shape": [ 28, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "adc7fd999de1c34f9ca31a3d93b5f0a159b3c9ff4d6d8ec10bcf843b21f3f38b", "shape": [ 7 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a87893910c503ca82fca06d9c0032304853a7ad9a6b99559819b7f43eb87d121", "shape": [ 28, 7 ] } ], "region": "r:830", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 28 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:798", "inputs": [ "s:799", "s:810" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:829", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 2.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:799", "inputs": [ "s:800", "s:801" ], "name": null, "op": "Add", "parameters": [], "region": "r:831", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:800", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:832", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:801", "inputs": [ "s:802" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:834", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:802", "inputs": [ "s:803" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:835", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:803", "inputs": [ "s:804" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:838", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:804", "inputs": [ "s:805" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:839", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:805", "inputs": [ "s:806", "s:807" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:839", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:806", "inputs": [ "s:800" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:841", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.9956489921875, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:807", "inputs": [ "s:808" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:843", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:808", "inputs": [ "s:809", "s:809", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:844", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:809", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:844", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:810", "inputs": [ "s:799" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:845", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 2.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:811", "inputs": [ "s:812" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:847", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:812", "inputs": [ "s:74", "s:813" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:846", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:813", "inputs": [ "s:814" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "67bbf7e8bef0cb24fa16b53fe5abf8ad64ff6177c3a9c829199acc4d3861b5ce", "shape": [ 61, 33 ] } ], "region": "r:848", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:814", "inputs": [ "s:815" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:850", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:815", "inputs": [ "s:816" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:851", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:816", "inputs": [ "s:817" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:853", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:817", "inputs": [ "s:818", "s:822" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:852", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 52.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:818", "inputs": [ "s:391", "s:819" ], "name": null, "op": "Add", "parameters": [], "region": "r:854", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 51.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:819", "inputs": [ "s:820" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "region": "r:855", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:820", "inputs": [ "s:821", "s:821", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "4245ea3ab317856147a0e2b3bcc47cf6cde638f5c262235ba777f4a403c34fae", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:856", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:821", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:856", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:822", "inputs": [ "s:818" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:857", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 52.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:823", "inputs": [ "s:824" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:859", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:824", "inputs": [ "s:825", "s:831" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:858", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:825", "inputs": [ "s:826" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:862", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:826", "inputs": [ "s:827" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:863", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:827", "inputs": [ "s:828", "s:829" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:863", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:828", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "583ebb6a4984ea315508e66cd81f1c0216a0719fc08f365cca73818d9552326c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:865", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:829", "inputs": [ "s:830" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "790ba56c4d91c3c94f262121ff631b25334504dc16753aed3fd7d9dc2803db41", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:867", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:830", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:868", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:831", "inputs": [ "s:832" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:870", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:832", "inputs": [ "s:833", "s:852", "s:871" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:870", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:833", "inputs": [ "s:834" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:872", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:834", "inputs": [ "s:37", "s:835" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:871", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:835", "inputs": [ "s:836" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc0a03a8e9c3e933278bf7f2667ac150ebffc7b84c9d541ff71e6243a22d70ca", "shape": [ 7, 33 ] } ], "region": "r:873", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:836", "inputs": [ "s:837" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "3addfb141cd7c9c4c6543a82191a3707ac29c7a041217782e61d4d91c691aee8", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "0a2ff5f661b6232dedd8c1bb0416601df78831ad504c44bd61acd771d4cbdd95", "shape": [ 7, 7 ] } ], "region": "r:875", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:837", "inputs": [ "s:838" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6f3a1a9d59ae648b4f66e10802905ef54f901e2409b9e01e653a05fd7c4ff786", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "68e014055f97f84bdfabe73267c9ca5a45445840d62edf66ebb0ad398c1d088d", "shape": [ 7, 7 ] } ], "region": "r:876", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:838", "inputs": [ "s:839" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ecd290af9b878d500003aada26fc578ffe18fb71ca439df97b2c5da3577bb637", "shape": [ 28 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ac723062162c7c0744a28c7316bd9750d4304fa46e93681ca5ee8499e8dd7cc9", "shape": [ 28, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "adc7fd999de1c34f9ca31a3d93b5f0a159b3c9ff4d6d8ec10bcf843b21f3f38b", "shape": [ 7 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a87893910c503ca82fca06d9c0032304853a7ad9a6b99559819b7f43eb87d121", "shape": [ 28, 7 ] } ], "region": "r:878", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 28 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:839", "inputs": [ "s:840", "s:851" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:877", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:840", "inputs": [ "s:841", "s:842" ], "name": null, "op": "Add", "parameters": [], "region": "r:879", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:841", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:880", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:842", "inputs": [ "s:843" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:882", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:843", "inputs": [ "s:844" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:883", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:844", "inputs": [ "s:845" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:886", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:845", "inputs": [ "s:846" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:887", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:846", "inputs": [ "s:847", "s:848" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:887", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:847", "inputs": [ "s:841" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:889", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:848", "inputs": [ "s:849" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:891", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:849", "inputs": [ "s:850", "s:850", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:892", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:850", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:892", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:851", "inputs": [ "s:840" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:893", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:852", "inputs": [ "s:853" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:895", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:853", "inputs": [ "s:65", "s:854" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:894", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:854", "inputs": [ "s:855" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc0a03a8e9c3e933278bf7f2667ac150ebffc7b84c9d541ff71e6243a22d70ca", "shape": [ 7, 33 ] } ], "region": "r:896", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:855", "inputs": [ "s:856" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "3addfb141cd7c9c4c6543a82191a3707ac29c7a041217782e61d4d91c691aee8", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "0a2ff5f661b6232dedd8c1bb0416601df78831ad504c44bd61acd771d4cbdd95", "shape": [ 7, 7 ] } ], "region": "r:898", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:856", "inputs": [ "s:857" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6f3a1a9d59ae648b4f66e10802905ef54f901e2409b9e01e653a05fd7c4ff786", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "68e014055f97f84bdfabe73267c9ca5a45445840d62edf66ebb0ad398c1d088d", "shape": [ 7, 7 ] } ], "region": "r:899", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:857", "inputs": [ "s:858" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ecd290af9b878d500003aada26fc578ffe18fb71ca439df97b2c5da3577bb637", "shape": [ 28 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ac723062162c7c0744a28c7316bd9750d4304fa46e93681ca5ee8499e8dd7cc9", "shape": [ 28, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "adc7fd999de1c34f9ca31a3d93b5f0a159b3c9ff4d6d8ec10bcf843b21f3f38b", "shape": [ 7 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a87893910c503ca82fca06d9c0032304853a7ad9a6b99559819b7f43eb87d121", "shape": [ 28, 7 ] } ], "region": "r:901", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 28 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:858", "inputs": [ "s:859", "s:870" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:900", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:859", "inputs": [ "s:860", "s:861" ], "name": null, "op": "Add", "parameters": [], "region": "r:902", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:860", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:903", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:861", "inputs": [ "s:862" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:905", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:862", "inputs": [ "s:863" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:906", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:863", "inputs": [ "s:864" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:909", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:864", "inputs": [ "s:865" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:910", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:865", "inputs": [ "s:866", "s:867" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:910", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:866", "inputs": [ "s:860" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:912", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:867", "inputs": [ "s:868" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:914", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:868", "inputs": [ "s:869", "s:869", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:915", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:869", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:915", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:870", "inputs": [ "s:859" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:916", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:871", "inputs": [ "s:872" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:918", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:872", "inputs": [ "s:74", "s:873" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:917", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:873", "inputs": [ "s:874" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "67bbf7e8bef0cb24fa16b53fe5abf8ad64ff6177c3a9c829199acc4d3861b5ce", "shape": [ 61, 33 ] } ], "region": "r:919", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:874", "inputs": [ "s:875" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:921", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:875", "inputs": [ "s:876" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:922", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:876", "inputs": [ "s:877" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:924", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:877", "inputs": [ "s:878", "s:882" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:923", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 34.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:878", "inputs": [ "s:345", "s:879" ], "name": null, "op": "Add", "parameters": [], "region": "r:925", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:879", "inputs": [ "s:880" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "95f9fcb4679410f50d029681502e3a2226e727c34a4fbf82de31c2d5c3420306", "shape": [ 33, 1 ] } ], "region": "r:926", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:880", "inputs": [ "s:881", "s:881", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2f89c627454e16505ffd9b5129358e251fa7ee409a68400e1f77bed409fc51e8", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:927", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:881", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:927", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:882", "inputs": [ "s:878" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:928", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 34.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:883", "inputs": [ "s:884" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:930", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:884", "inputs": [ "s:885", "s:891" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:929", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:885", "inputs": [ "s:886" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:933", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:886", "inputs": [ "s:887" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:934", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:887", "inputs": [ "s:888", "s:889" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:934", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:888", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "418aec4f616c4d163c2123ca102e0033c51a01409d61fd37b527cb01c86d1366", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:936", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:889", "inputs": [ "s:890" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f41aacfe564dfc54191d249d1cd466a118145c3f617069e8e30958f4f57cf340", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:938", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:890", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:939", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:891", "inputs": [ "s:892" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:941", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:892", "inputs": [ "s:893", "s:912", "s:931" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:941", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:893", "inputs": [ "s:894" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:943", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:894", "inputs": [ "s:37", "s:895" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:942", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:895", "inputs": [ "s:896" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc0a03a8e9c3e933278bf7f2667ac150ebffc7b84c9d541ff71e6243a22d70ca", "shape": [ 7, 33 ] } ], "region": "r:944", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:896", "inputs": [ "s:897" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "3addfb141cd7c9c4c6543a82191a3707ac29c7a041217782e61d4d91c691aee8", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "0a2ff5f661b6232dedd8c1bb0416601df78831ad504c44bd61acd771d4cbdd95", "shape": [ 7, 7 ] } ], "region": "r:946", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:897", "inputs": [ "s:898" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6f3a1a9d59ae648b4f66e10802905ef54f901e2409b9e01e653a05fd7c4ff786", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "68e014055f97f84bdfabe73267c9ca5a45445840d62edf66ebb0ad398c1d088d", "shape": [ 7, 7 ] } ], "region": "r:947", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:898", "inputs": [ "s:899" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ecd290af9b878d500003aada26fc578ffe18fb71ca439df97b2c5da3577bb637", "shape": [ 28 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ac723062162c7c0744a28c7316bd9750d4304fa46e93681ca5ee8499e8dd7cc9", "shape": [ 28, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "adc7fd999de1c34f9ca31a3d93b5f0a159b3c9ff4d6d8ec10bcf843b21f3f38b", "shape": [ 7 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a87893910c503ca82fca06d9c0032304853a7ad9a6b99559819b7f43eb87d121", "shape": [ 28, 7 ] } ], "region": "r:949", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 28 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:899", "inputs": [ "s:900", "s:911" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:948", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:900", "inputs": [ "s:901", "s:902" ], "name": null, "op": "Add", "parameters": [], "region": "r:950", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:901", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:951", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:902", "inputs": [ "s:903" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:953", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:903", "inputs": [ "s:904" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:954", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:904", "inputs": [ "s:905" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:957", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:905", "inputs": [ "s:906" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:958", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:906", "inputs": [ "s:907", "s:908" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:958", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:907", "inputs": [ "s:901" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:960", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:908", "inputs": [ "s:909" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:962", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:909", "inputs": [ "s:910", "s:910", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:963", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:910", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:963", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:911", "inputs": [ "s:900" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:964", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:912", "inputs": [ "s:913" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:966", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:913", "inputs": [ "s:65", "s:914" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:965", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:914", "inputs": [ "s:915" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc0a03a8e9c3e933278bf7f2667ac150ebffc7b84c9d541ff71e6243a22d70ca", "shape": [ 7, 33 ] } ], "region": "r:967", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:915", "inputs": [ "s:916" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "3addfb141cd7c9c4c6543a82191a3707ac29c7a041217782e61d4d91c691aee8", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "0a2ff5f661b6232dedd8c1bb0416601df78831ad504c44bd61acd771d4cbdd95", "shape": [ 7, 7 ] } ], "region": "r:969", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:916", "inputs": [ "s:917" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6f3a1a9d59ae648b4f66e10802905ef54f901e2409b9e01e653a05fd7c4ff786", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "68e014055f97f84bdfabe73267c9ca5a45445840d62edf66ebb0ad398c1d088d", "shape": [ 7, 7 ] } ], "region": "r:970", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:917", "inputs": [ "s:918" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ecd290af9b878d500003aada26fc578ffe18fb71ca439df97b2c5da3577bb637", "shape": [ 28 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ac723062162c7c0744a28c7316bd9750d4304fa46e93681ca5ee8499e8dd7cc9", "shape": [ 28, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "adc7fd999de1c34f9ca31a3d93b5f0a159b3c9ff4d6d8ec10bcf843b21f3f38b", "shape": [ 7 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a87893910c503ca82fca06d9c0032304853a7ad9a6b99559819b7f43eb87d121", "shape": [ 28, 7 ] } ], "region": "r:972", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 28 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:918", "inputs": [ "s:919", "s:930" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:971", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:919", "inputs": [ "s:920", "s:921" ], "name": null, "op": "Add", "parameters": [], "region": "r:973", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:920", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:974", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:921", "inputs": [ "s:922" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:976", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:922", "inputs": [ "s:923" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:977", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:923", "inputs": [ "s:924" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:980", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:924", "inputs": [ "s:925" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:981", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:925", "inputs": [ "s:926", "s:927" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:981", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:926", "inputs": [ "s:920" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:983", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:927", "inputs": [ "s:928" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:985", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:928", "inputs": [ "s:929", "s:929", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:986", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:929", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:986", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:930", "inputs": [ "s:919" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:987", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:931", "inputs": [ "s:932" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:989", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:932", "inputs": [ "s:74", "s:933" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:988", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:933", "inputs": [ "s:934" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "67bbf7e8bef0cb24fa16b53fe5abf8ad64ff6177c3a9c829199acc4d3861b5ce", "shape": [ 61, 33 ] } ], "region": "r:990", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:934", "inputs": [ "s:935" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "b33e76357c515a25091049d146d7acf2ace055261b0fb28aff0572f8c448efb6", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "cf9e38fef24b6ed2c00ea1f63aa957688d382211541f268febf337faefcd1e29", "shape": [ 61, 61 ] } ], "region": "r:992", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:935", "inputs": [ "s:936" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "525a1abd016736a6b07bf907ef259e0d78ce9993bdaffb3b87d0bc9e30d43d3f", "shape": [ 61 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "70f30da7058136218c4c3af5dbb397f40a8f10458277baa20ca579102d22717a", "shape": [ 61, 61 ] } ], "region": "r:993", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:936", "inputs": [ "s:937" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "29e6437fd1d5e3da4e21844f356188af719d4d1593f105ca0300472094c5b61e", "shape": [ 244 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "07db4ef2846650a4df1813e64a9bf0effad4b919bed2fe32925fda185a4339e2", "shape": [ 244, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "2f1e437e03de0a9d954b829b7cd95131f7e59884184143541b7e2194c7895d5a", "shape": [ 61 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "2c3208ceb4f80f7654c103057cf2735dfdeb5e19f5fd17a5496070162ef594ce", "shape": [ 244, 61 ] } ], "region": "r:995", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 244 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 61 }, { "annotation": null, "id": "s:937", "inputs": [ "s:938", "s:940" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:994", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 10.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:938", "inputs": [ "s:270", "s:939" ], "name": null, "op": "Add", "parameters": [], "region": "r:996", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 9.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:939", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:997", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:940", "inputs": [ "s:938" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:998", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 10.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:941", "inputs": [ "s:942" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1000", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:942", "inputs": [ "s:943", "s:949" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:999", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:943", "inputs": [ "s:944" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1003", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:944", "inputs": [ "s:945" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1004", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:945", "inputs": [ "s:946", "s:947" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1004", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:946", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "3de0afcfd401c63e8410995e0ed14091535fa844c2124bc3eee71147919929ba", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1006", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:947", "inputs": [ "s:948" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "e199fd05223bf62f88cccb9b9409bfa30c64669d538c5fe324439c03e8718de9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1008", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:948", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1009", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:949", "inputs": [ "s:950" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1011", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:950", "inputs": [ "s:951", "s:954", "s:957" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1011", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:951", "inputs": [ "s:952" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1013", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:952", "inputs": [ "s:37", "s:953" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1012", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:953", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d3d4b4498b152d62ffae25a2127656848110f4397c0385d2fa75457e20fbbf30", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "region": "r:1014", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:954", "inputs": [ "s:955" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1016", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:955", "inputs": [ "s:65", "s:956" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1015", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:956", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d3d4b4498b152d62ffae25a2127656848110f4397c0385d2fa75457e20fbbf30", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "region": "r:1017", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:957", "inputs": [ "s:958" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1019", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:958", "inputs": [ "s:74", "s:959" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1018", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:959", "inputs": [ "s:960" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc0a03a8e9c3e933278bf7f2667ac150ebffc7b84c9d541ff71e6243a22d70ca", "shape": [ 7, 33 ] } ], "region": "r:1020", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:960", "inputs": [ "s:961" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "3addfb141cd7c9c4c6543a82191a3707ac29c7a041217782e61d4d91c691aee8", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "0a2ff5f661b6232dedd8c1bb0416601df78831ad504c44bd61acd771d4cbdd95", "shape": [ 7, 7 ] } ], "region": "r:1022", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:961", "inputs": [ "s:962" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6f3a1a9d59ae648b4f66e10802905ef54f901e2409b9e01e653a05fd7c4ff786", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "68e014055f97f84bdfabe73267c9ca5a45445840d62edf66ebb0ad398c1d088d", "shape": [ 7, 7 ] } ], "region": "r:1023", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:962", "inputs": [ "s:963" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ecd290af9b878d500003aada26fc578ffe18fb71ca439df97b2c5da3577bb637", "shape": [ 28 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ac723062162c7c0744a28c7316bd9750d4304fa46e93681ca5ee8499e8dd7cc9", "shape": [ 28, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "adc7fd999de1c34f9ca31a3d93b5f0a159b3c9ff4d6d8ec10bcf843b21f3f38b", "shape": [ 7 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a87893910c503ca82fca06d9c0032304853a7ad9a6b99559819b7f43eb87d121", "shape": [ 28, 7 ] } ], "region": "r:1025", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 28 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:963", "inputs": [ "s:964", "s:975" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1024", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 2.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:964", "inputs": [ "s:965", "s:966" ], "name": null, "op": "Add", "parameters": [], "region": "r:1026", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:965", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:1027", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:966", "inputs": [ "s:967" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:1029", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:967", "inputs": [ "s:968" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1030", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:968", "inputs": [ "s:969" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1033", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:969", "inputs": [ "s:970" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1034", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:970", "inputs": [ "s:971", "s:972" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1034", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:971", "inputs": [ "s:965" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1036", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.9956489921875, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:972", "inputs": [ "s:973" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:1038", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:973", "inputs": [ "s:974", "s:974", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1039", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:974", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1039", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:975", "inputs": [ "s:964" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1040", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 2.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:976", "inputs": [ "s:977" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1042", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0250499999999994))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:977", "inputs": [ "s:978", "s:984" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1041", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:978", "inputs": [ "s:979" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1045", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:979", "inputs": [ "s:980" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1046", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:980", "inputs": [ "s:981", "s:982" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1046", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:981", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "52991e853857f7561b6157d0b54b4a07ae9c0e669754333e62de05736841660b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1048", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:982", "inputs": [ "s:983" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c63332a208ca2aa557eddae88226b8fc2a8d2adbff8f79f7fc3f3808bc062c9f", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1050", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:983", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1051", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:984", "inputs": [ "s:985" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1053", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:985", "inputs": [ "s:986", "s:1011", "s:1044" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1053", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:986", "inputs": [ "s:987" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1055", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:987", "inputs": [ "s:37", "s:988" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1054", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:988", "inputs": [ "s:989" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1057", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:989", "inputs": [ "s:990", "s:998", "s:999" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1056", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:990", "inputs": [ "s:991" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1059", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:991", "inputs": [ "s:992" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1060", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0021755078125, "lower_unbounded": false, "upper": 7.504351015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:992", "inputs": [ "s:993" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1062", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0021755078125, "lower_unbounded": false, "upper": 7.504351015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:993", "inputs": [ "s:994", "s:997" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1061", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:994", "inputs": [ "s:995" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1063", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:995", "inputs": [ "s:996", "s:996", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1064", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:996", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1064", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:997", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "ea2845900b5856c9bf354b1aa9761b5aa6888e5ed61738fe9579ca42bc0f6054", "shape": [ 1 ], "values": [ 3.0 ] } ], "region": "r:1065", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 3.0, "lower_unbounded": false, "upper": 3.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:998", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1066", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:999", "inputs": [ "s:1000", "s:1005", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1067", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1000", "inputs": [ "s:1001" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9f7ce09d183259d69fb622b5248bdb85f5f8493e54529def7b337f69760ed024", "shape": [ 4, 4 ] } ], "region": "r:1069", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1001", "inputs": [ "s:1002" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "f6bb1294da2f78cd935b01c7656280df5eaa0439e9d97bc03775825a41a508e4", "shape": [ 4 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "5998e9d1dd9bf48af6af87a1ef8c7783b818ac8329637ada08cc5fb329042376", "shape": [ 4, 4 ] } ], "region": "r:1070", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1002", "inputs": [ "s:1003" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9017d307c9bd1ac42160d5fae5ff3c4ea7782dd5541b1e83341861e134cae614", "shape": [ 16 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "e4c555906ab7dff40f54eb6916d8ede133b922f31fbcd7f2d25bd787286bf7d3", "shape": [ 16, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e74125e44e9343a83c86395ec81731b98f9aefbe35b61529fe730de3c82a8aab", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e14367196378c02a631dc1b99191297e21c25fac83ce07c8ff35070293c44ccd", "shape": [ 16, 4 ] } ], "region": "r:1072", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 16 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1003", "inputs": [ "s:991", "s:1004" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1071", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0021755078125, "lower_unbounded": false, "upper": 8.504351015625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1004", "inputs": [ "s:991" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1073", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0021755078125, "lower_unbounded": false, "upper": 8.504351015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1005", "inputs": [ "s:1006" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9f7ce09d183259d69fb622b5248bdb85f5f8493e54529def7b337f69760ed024", "shape": [ 4, 4 ] } ], "region": "r:1075", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1006", "inputs": [ "s:1007" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "f6bb1294da2f78cd935b01c7656280df5eaa0439e9d97bc03775825a41a508e4", "shape": [ 4 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "5998e9d1dd9bf48af6af87a1ef8c7783b818ac8329637ada08cc5fb329042376", "shape": [ 4, 4 ] } ], "region": "r:1076", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1007", "inputs": [ "s:1008" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9017d307c9bd1ac42160d5fae5ff3c4ea7782dd5541b1e83341861e134cae614", "shape": [ 16 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "e4c555906ab7dff40f54eb6916d8ede133b922f31fbcd7f2d25bd787286bf7d3", "shape": [ 16, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e74125e44e9343a83c86395ec81731b98f9aefbe35b61529fe730de3c82a8aab", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e14367196378c02a631dc1b99191297e21c25fac83ce07c8ff35070293c44ccd", "shape": [ 16, 4 ] } ], "region": "r:1078", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 16 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1008", "inputs": [ "s:1009", "s:1010" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1077", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -6.357438532905364, "lower_unbounded": false, "upper": 24.52410519957203, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1009", "inputs": [ "s:18" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6bd5e30e99b6cfe9c9e85bcbe7ae22cda0df1fb6f5c858c4448e5c127424c7f4", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1079", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -6.357438532905364, "lower_unbounded": false, "upper": 23.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1010", "inputs": [ "s:1009" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1080", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -5.357438532905364, "lower_unbounded": false, "upper": 24.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1011", "inputs": [ "s:1012" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1082", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1012", "inputs": [ "s:65", "s:1013" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1081", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1013", "inputs": [ "s:1014" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1084", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1014", "inputs": [ "s:1015", "s:1019", "s:1043" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1083", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:1015", "inputs": [ "s:1016" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1086", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1016", "inputs": [ "s:1017" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "region": "r:1087", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1017", "inputs": [ "s:1018", "s:1018", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a3eaf4f37ccad625864abeda4df5d0fce93c2f3d0f30ade108a13425a5ded31a", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1088", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1018", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1088", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1019", "inputs": [ "s:1020", "s:1037", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "be94494d5c98aa4e14ad64130eab9a1dc7a79a46a59d9e84d353075468c0a9fd", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "59cc5d44d3cb84eddef86ad62b93e827db0d8a54aa8a8364814b8a098a8c6168", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1089", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1020", "inputs": [ "s:1021" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:1091", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1021", "inputs": [ "s:1022" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:1092", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1022", "inputs": [ "s:1023" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:1094", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1023", "inputs": [ "s:1024", "s:1036" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1093", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -5.00435101171875, "lower_unbounded": false, "upper": 8.339859852864585, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1024", "inputs": [ "s:1025", "s:1032" ], "name": null, "op": "Add", "parameters": [], "region": "r:1096", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -5.00435101171875, "lower_unbounded": false, "upper": 7.339859852864584, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1025", "inputs": [ "s:1026" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1097", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.0021755078125, "lower_unbounded": false, "upper": 7.337684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1026", "inputs": [ "s:1027" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1099", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.0021755078125, "lower_unbounded": false, "upper": 7.337684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1027", "inputs": [ "s:1028", "s:1031" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1098", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1028", "inputs": [ "s:1029" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1100", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1029", "inputs": [ "s:1030", "s:1030", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1101", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1030", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1101", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1031", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d88c86f15bbea365d658ad95a81d45367c465f7af6f7264fb077f01747ddc77d", "shape": [ 1 ], "values": [ 2.0 ] } ], "region": "r:1102", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 2.0, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1032", "inputs": [ "s:1033" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1103", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755039062499, "lower_unbounded": false, "upper": 0.002175503906249987, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1033", "inputs": [ "s:1034" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:1105", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1034", "inputs": [ "s:1035" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1106", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1035", "inputs": [ "s:1015" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:1109", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1036", "inputs": [ "s:1024" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1110", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.00435101171875, "lower_unbounded": false, "upper": 8.339859852864585, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1037", "inputs": [ "s:1038" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:1112", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1038", "inputs": [ "s:1039" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:1113", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1039", "inputs": [ "s:1040" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:1115", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1040", "inputs": [ "s:1041", "s:1042" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1114", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -8.357438532905364, "lower_unbounded": false, "upper": 22.52410519957203, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1041", "inputs": [ "s:18" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "053ac09e0109c6d84088a0106d301d00ebe129455cb7889c911000d6f462271f", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1116", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -8.357438532905364, "lower_unbounded": false, "upper": 21.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1042", "inputs": [ "s:1041" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1117", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -7.357438532905364, "lower_unbounded": false, "upper": 22.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1043", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "44250fce6a5c41d7a7e24f8100dbb6befdf5b44ab7d94fe23f9cec9ea6693f46", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1118", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1044", "inputs": [ "s:1045" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1120", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1045", "inputs": [ "s:74", "s:1046" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1119", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1046", "inputs": [ "s:1047" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc0a03a8e9c3e933278bf7f2667ac150ebffc7b84c9d541ff71e6243a22d70ca", "shape": [ 7, 33 ] } ], "region": "r:1121", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1047", "inputs": [ "s:1048" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "3addfb141cd7c9c4c6543a82191a3707ac29c7a041217782e61d4d91c691aee8", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "0a2ff5f661b6232dedd8c1bb0416601df78831ad504c44bd61acd771d4cbdd95", "shape": [ 7, 7 ] } ], "region": "r:1123", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1048", "inputs": [ "s:1049" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6f3a1a9d59ae648b4f66e10802905ef54f901e2409b9e01e653a05fd7c4ff786", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "68e014055f97f84bdfabe73267c9ca5a45445840d62edf66ebb0ad398c1d088d", "shape": [ 7, 7 ] } ], "region": "r:1124", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1049", "inputs": [ "s:1050" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ecd290af9b878d500003aada26fc578ffe18fb71ca439df97b2c5da3577bb637", "shape": [ 28 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ac723062162c7c0744a28c7316bd9750d4304fa46e93681ca5ee8499e8dd7cc9", "shape": [ 28, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "adc7fd999de1c34f9ca31a3d93b5f0a159b3c9ff4d6d8ec10bcf843b21f3f38b", "shape": [ 7 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a87893910c503ca82fca06d9c0032304853a7ad9a6b99559819b7f43eb87d121", "shape": [ 28, 7 ] } ], "region": "r:1126", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 28 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1050", "inputs": [ "s:1051", "s:1062" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1125", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1051", "inputs": [ "s:1052", "s:1053" ], "name": null, "op": "Add", "parameters": [], "region": "r:1127", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1052", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1128", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1053", "inputs": [ "s:1054" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:1130", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1054", "inputs": [ "s:1055" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1131", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1055", "inputs": [ "s:1056" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1134", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1056", "inputs": [ "s:1057" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1135", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1057", "inputs": [ "s:1058", "s:1059" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1135", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1058", "inputs": [ "s:1052" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1137", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1059", "inputs": [ "s:1060" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:1139", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1060", "inputs": [ "s:1061", "s:1061", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1140", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1061", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1140", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1062", "inputs": [ "s:1051" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1141", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1063", "inputs": [ "s:1064" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1143", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0250499999999994))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1064", "inputs": [ "s:1065", "s:1071" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1142", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1065", "inputs": [ "s:1066" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1146", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1066", "inputs": [ "s:1067" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1147", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1067", "inputs": [ "s:1068", "s:1069" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1147", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1068", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ec72bdc48ad6906fca6435b50f45617abf02e58e2863796f068e2d57eca6f101", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1149", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1069", "inputs": [ "s:1070" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "d35180f5b6f03da9db760a267ff605e9cc819de41520fc22aacc937ca0e23463", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1151", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1070", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1152", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1071", "inputs": [ "s:1072" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1154", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1072", "inputs": [ "s:1073", "s:1091", "s:1119" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1154", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:1073", "inputs": [ "s:1074" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1156", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1074", "inputs": [ "s:37", "s:1075" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1155", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1075", "inputs": [ "s:1076" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1158", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1076", "inputs": [ "s:1077", "s:998", "s:1085" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1157", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:1077", "inputs": [ "s:1078" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1160", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1078", "inputs": [ "s:1079" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1161", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0021755078125, "lower_unbounded": false, "upper": 8.504351015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1079", "inputs": [ "s:1080" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1163", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0021755078125, "lower_unbounded": false, "upper": 7.504351015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1080", "inputs": [ "s:1081", "s:1084" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1162", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1081", "inputs": [ "s:1082" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1164", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1082", "inputs": [ "s:1083", "s:1083", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1165", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1083", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1165", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1084", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "ea2845900b5856c9bf354b1aa9761b5aa6888e5ed61738fe9579ca42bc0f6054", "shape": [ 1 ], "values": [ 3.0 ] } ], "region": "r:1166", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 3.0, "lower_unbounded": false, "upper": 3.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1085", "inputs": [ "s:1086", "s:1005", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1167", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1086", "inputs": [ "s:1087" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9f7ce09d183259d69fb622b5248bdb85f5f8493e54529def7b337f69760ed024", "shape": [ 4, 4 ] } ], "region": "r:1169", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1087", "inputs": [ "s:1088" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "f6bb1294da2f78cd935b01c7656280df5eaa0439e9d97bc03775825a41a508e4", "shape": [ 4 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "5998e9d1dd9bf48af6af87a1ef8c7783b818ac8329637ada08cc5fb329042376", "shape": [ 4, 4 ] } ], "region": "r:1170", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1088", "inputs": [ "s:1089" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9017d307c9bd1ac42160d5fae5ff3c4ea7782dd5541b1e83341861e134cae614", "shape": [ 16 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "e4c555906ab7dff40f54eb6916d8ede133b922f31fbcd7f2d25bd787286bf7d3", "shape": [ 16, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e74125e44e9343a83c86395ec81731b98f9aefbe35b61529fe730de3c82a8aab", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e14367196378c02a631dc1b99191297e21c25fac83ce07c8ff35070293c44ccd", "shape": [ 16, 4 ] } ], "region": "r:1172", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 16 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1089", "inputs": [ "s:1078", "s:1090" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1171", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0021755078125, "lower_unbounded": false, "upper": 9.504351015625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1090", "inputs": [ "s:1078" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1173", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755078124999, "lower_unbounded": false, "upper": 9.504351015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1091", "inputs": [ "s:1092" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1175", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1092", "inputs": [ "s:65", "s:1093" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1174", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1093", "inputs": [ "s:1094" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1177", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1094", "inputs": [ "s:1095", "s:1112", "s:1113" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1176", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:1095", "inputs": [ "s:1096" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1179", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1096", "inputs": [ "s:1097", "s:1104" ], "name": null, "op": "Add", "parameters": [], "region": "r:1181", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.00435101171875, "lower_unbounded": false, "upper": 8.339859852864583, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1097", "inputs": [ "s:1098" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1182", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0021755078125, "lower_unbounded": false, "upper": 8.337684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1098", "inputs": [ "s:1099" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1184", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.0021755078125, "lower_unbounded": false, "upper": 7.337684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1099", "inputs": [ "s:1100", "s:1103" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1183", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1100", "inputs": [ "s:1101" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1185", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1101", "inputs": [ "s:1102", "s:1102", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1186", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1102", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1186", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1103", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d88c86f15bbea365d658ad95a81d45367c465f7af6f7264fb077f01747ddc77d", "shape": [ 1 ], "values": [ 2.0 ] } ], "region": "r:1187", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 2.0, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1104", "inputs": [ "s:1105" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1188", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755039062499, "lower_unbounded": false, "upper": 0.002175503906249987, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1105", "inputs": [ "s:1106" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:1190", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1106", "inputs": [ "s:1107" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1191", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1107", "inputs": [ "s:1108" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:1194", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1108", "inputs": [ "s:1109" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1196", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1109", "inputs": [ "s:1110" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "region": "r:1197", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1110", "inputs": [ "s:1111", "s:1111", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "64b0a4812c027fb37ce2b91630d9df5c6549ab2445a912d1b8ded79071f00141", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1198", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1111", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1198", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1112", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1199", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1113", "inputs": [ "s:1114", "s:1037", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "be94494d5c98aa4e14ad64130eab9a1dc7a79a46a59d9e84d353075468c0a9fd", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "59cc5d44d3cb84eddef86ad62b93e827db0d8a54aa8a8364814b8a098a8c6168", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1200", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1114", "inputs": [ "s:1115" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:1202", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1115", "inputs": [ "s:1116" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:1203", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1116", "inputs": [ "s:1117" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:1205", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1117", "inputs": [ "s:1096", "s:1118" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1204", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.00435101171875, "lower_unbounded": false, "upper": 9.339859852864583, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1118", "inputs": [ "s:1096" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1206", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.00435101171875, "lower_unbounded": false, "upper": 9.339859852864583, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1119", "inputs": [ "s:1120" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1208", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1120", "inputs": [ "s:74", "s:1121" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1207", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1121", "inputs": [ "s:1122" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc0a03a8e9c3e933278bf7f2667ac150ebffc7b84c9d541ff71e6243a22d70ca", "shape": [ 7, 33 ] } ], "region": "r:1209", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1122", "inputs": [ "s:1123" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "3addfb141cd7c9c4c6543a82191a3707ac29c7a041217782e61d4d91c691aee8", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "0a2ff5f661b6232dedd8c1bb0416601df78831ad504c44bd61acd771d4cbdd95", "shape": [ 7, 7 ] } ], "region": "r:1211", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1123", "inputs": [ "s:1124" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6f3a1a9d59ae648b4f66e10802905ef54f901e2409b9e01e653a05fd7c4ff786", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "68e014055f97f84bdfabe73267c9ca5a45445840d62edf66ebb0ad398c1d088d", "shape": [ 7, 7 ] } ], "region": "r:1212", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1124", "inputs": [ "s:1125" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ecd290af9b878d500003aada26fc578ffe18fb71ca439df97b2c5da3577bb637", "shape": [ 28 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ac723062162c7c0744a28c7316bd9750d4304fa46e93681ca5ee8499e8dd7cc9", "shape": [ 28, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "adc7fd999de1c34f9ca31a3d93b5f0a159b3c9ff4d6d8ec10bcf843b21f3f38b", "shape": [ 7 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a87893910c503ca82fca06d9c0032304853a7ad9a6b99559819b7f43eb87d121", "shape": [ 28, 7 ] } ], "region": "r:1214", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 28 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1125", "inputs": [ "s:1126", "s:1137" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1213", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1126", "inputs": [ "s:1127", "s:1128" ], "name": null, "op": "Add", "parameters": [], "region": "r:1215", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1127", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1216", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1128", "inputs": [ "s:1129" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:1218", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1129", "inputs": [ "s:1130" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1219", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1130", "inputs": [ "s:1131" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1222", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1131", "inputs": [ "s:1132" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1223", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1132", "inputs": [ "s:1133", "s:1134" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1223", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1133", "inputs": [ "s:1127" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1225", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1134", "inputs": [ "s:1135" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:1227", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1135", "inputs": [ "s:1136", "s:1136", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1228", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1136", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1228", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1137", "inputs": [ "s:1126" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1229", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1138", "inputs": [ "s:1139" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1231", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0250499999999994))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1139", "inputs": [ "s:1140", "s:1146" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1230", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1140", "inputs": [ "s:1141" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1234", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1141", "inputs": [ "s:1142" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1235", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1142", "inputs": [ "s:1143", "s:1144" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1235", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1143", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f168e05b01b0f8f0195051bd5bbc1643e3448dcb28d39baa7f9ee0dc6482cb03", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1237", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1144", "inputs": [ "s:1145" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c152adf65b2187fcc340d0af789d32925fae822758408a76032143d93dcf2f", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1239", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1145", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1240", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1146", "inputs": [ "s:1147" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1242", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1147", "inputs": [ "s:1148", "s:1166", "s:1193" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1242", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:1148", "inputs": [ "s:1149" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1244", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1149", "inputs": [ "s:37", "s:1150" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1243", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1150", "inputs": [ "s:1151" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1246", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1151", "inputs": [ "s:1152", "s:998", "s:1160" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1245", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:1152", "inputs": [ "s:1153" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1248", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1153", "inputs": [ "s:1154" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d88c86f15bbea365d658ad95a81d45367c465f7af6f7264fb077f01747ddc77d", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1249", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755078124999, "lower_unbounded": false, "upper": 9.504351015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1154", "inputs": [ "s:1155" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1251", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0021755078125, "lower_unbounded": false, "upper": 7.504351015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1155", "inputs": [ "s:1156", "s:1159" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1250", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1156", "inputs": [ "s:1157" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1252", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1157", "inputs": [ "s:1158", "s:1158", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1253", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1158", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1253", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1159", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "ea2845900b5856c9bf354b1aa9761b5aa6888e5ed61738fe9579ca42bc0f6054", "shape": [ 1 ], "values": [ 3.0 ] } ], "region": "r:1254", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 3.0, "lower_unbounded": false, "upper": 3.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1160", "inputs": [ "s:1161", "s:1005", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1255", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1161", "inputs": [ "s:1162" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9f7ce09d183259d69fb622b5248bdb85f5f8493e54529def7b337f69760ed024", "shape": [ 4, 4 ] } ], "region": "r:1257", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1162", "inputs": [ "s:1163" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "f6bb1294da2f78cd935b01c7656280df5eaa0439e9d97bc03775825a41a508e4", "shape": [ 4 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "5998e9d1dd9bf48af6af87a1ef8c7783b818ac8329637ada08cc5fb329042376", "shape": [ 4, 4 ] } ], "region": "r:1258", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1163", "inputs": [ "s:1164" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9017d307c9bd1ac42160d5fae5ff3c4ea7782dd5541b1e83341861e134cae614", "shape": [ 16 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "e4c555906ab7dff40f54eb6916d8ede133b922f31fbcd7f2d25bd787286bf7d3", "shape": [ 16, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e74125e44e9343a83c86395ec81731b98f9aefbe35b61529fe730de3c82a8aab", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e14367196378c02a631dc1b99191297e21c25fac83ce07c8ff35070293c44ccd", "shape": [ 16, 4 ] } ], "region": "r:1260", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 16 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1164", "inputs": [ "s:1153", "s:1165" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1259", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755078124999, "lower_unbounded": false, "upper": 10.504351015625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1165", "inputs": [ "s:1153" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1261", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175507812499866, "lower_unbounded": false, "upper": 10.504351015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1166", "inputs": [ "s:1167" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1263", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1167", "inputs": [ "s:65", "s:1168" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1262", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1168", "inputs": [ "s:1169" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1265", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1169", "inputs": [ "s:1170", "s:1112", "s:1187" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1264", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:1170", "inputs": [ "s:1171" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1267", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1171", "inputs": [ "s:1172", "s:1179" ], "name": null, "op": "Add", "parameters": [], "region": "r:1269", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0043510117187497, "lower_unbounded": false, "upper": 9.339859852864583, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1172", "inputs": [ "s:1173" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d88c86f15bbea365d658ad95a81d45367c465f7af6f7264fb077f01747ddc77d", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1270", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0021755078125, "lower_unbounded": false, "upper": 9.337684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1173", "inputs": [ "s:1174" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1272", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.0021755078125, "lower_unbounded": false, "upper": 7.337684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1174", "inputs": [ "s:1175", "s:1178" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1271", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1175", "inputs": [ "s:1176" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1273", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1176", "inputs": [ "s:1177", "s:1177", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1274", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1177", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1274", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1178", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d88c86f15bbea365d658ad95a81d45367c465f7af6f7264fb077f01747ddc77d", "shape": [ 1 ], "values": [ 2.0 ] } ], "region": "r:1275", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 2.0, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1179", "inputs": [ "s:1180" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1276", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755039062499, "lower_unbounded": false, "upper": 0.002175503906249987, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1180", "inputs": [ "s:1181" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:1278", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1181", "inputs": [ "s:1182" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1279", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1182", "inputs": [ "s:1183" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:1282", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1183", "inputs": [ "s:1184" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1284", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1184", "inputs": [ "s:1185" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "region": "r:1285", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1185", "inputs": [ "s:1186", "s:1186", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "b21306b2ca4e61b9d9672dee3108ec3e685bfa6056de62cdb01cb85dc62821e9", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1286", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1186", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1286", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1187", "inputs": [ "s:1188", "s:1037", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "be94494d5c98aa4e14ad64130eab9a1dc7a79a46a59d9e84d353075468c0a9fd", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "59cc5d44d3cb84eddef86ad62b93e827db0d8a54aa8a8364814b8a098a8c6168", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1287", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1188", "inputs": [ "s:1189" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:1289", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1189", "inputs": [ "s:1190" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:1290", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1190", "inputs": [ "s:1191" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:1292", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1191", "inputs": [ "s:1171", "s:1192" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1291", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0043510117187497, "lower_unbounded": false, "upper": 10.339859852864583, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1192", "inputs": [ "s:1171" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1293", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0043510117187497, "lower_unbounded": false, "upper": 10.339859852864583, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1193", "inputs": [ "s:1194" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1295", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1194", "inputs": [ "s:74", "s:1195" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1294", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1195", "inputs": [ "s:1196" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc0a03a8e9c3e933278bf7f2667ac150ebffc7b84c9d541ff71e6243a22d70ca", "shape": [ 7, 33 ] } ], "region": "r:1296", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1196", "inputs": [ "s:1197" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "3addfb141cd7c9c4c6543a82191a3707ac29c7a041217782e61d4d91c691aee8", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "0a2ff5f661b6232dedd8c1bb0416601df78831ad504c44bd61acd771d4cbdd95", "shape": [ 7, 7 ] } ], "region": "r:1298", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1197", "inputs": [ "s:1198" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6f3a1a9d59ae648b4f66e10802905ef54f901e2409b9e01e653a05fd7c4ff786", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "68e014055f97f84bdfabe73267c9ca5a45445840d62edf66ebb0ad398c1d088d", "shape": [ 7, 7 ] } ], "region": "r:1299", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1198", "inputs": [ "s:1199" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ecd290af9b878d500003aada26fc578ffe18fb71ca439df97b2c5da3577bb637", "shape": [ 28 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ac723062162c7c0744a28c7316bd9750d4304fa46e93681ca5ee8499e8dd7cc9", "shape": [ 28, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "adc7fd999de1c34f9ca31a3d93b5f0a159b3c9ff4d6d8ec10bcf843b21f3f38b", "shape": [ 7 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a87893910c503ca82fca06d9c0032304853a7ad9a6b99559819b7f43eb87d121", "shape": [ 28, 7 ] } ], "region": "r:1301", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 28 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1199", "inputs": [ "s:1200", "s:1211" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1300", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1200", "inputs": [ "s:1201", "s:1202" ], "name": null, "op": "Add", "parameters": [], "region": "r:1302", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1201", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1303", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1202", "inputs": [ "s:1203" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:1305", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1203", "inputs": [ "s:1204" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1306", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1204", "inputs": [ "s:1205" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1309", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1205", "inputs": [ "s:1206" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1310", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1206", "inputs": [ "s:1207", "s:1208" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1310", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1207", "inputs": [ "s:1201" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1312", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1208", "inputs": [ "s:1209" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:1314", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1209", "inputs": [ "s:1210", "s:1210", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1315", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1210", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1315", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1211", "inputs": [ "s:1200" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1316", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1212", "inputs": [ "s:1213" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1318", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0250499999999994))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1213", "inputs": [ "s:1214", "s:1220" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1317", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1214", "inputs": [ "s:1215" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1321", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1215", "inputs": [ "s:1216" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1322", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1216", "inputs": [ "s:1217", "s:1218" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1322", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1217", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "cb6b0c11488519a6658b9e429b5a8ceedfd51a949605930741312ea58bbe797b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1324", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1218", "inputs": [ "s:1219" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "106bb444492286cece2c2f3471615ce3ed37466f891bf2df4c164be23d396f57", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1326", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1219", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1327", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1220", "inputs": [ "s:1221" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1329", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0250499999999994, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1221", "inputs": [ "s:1222", "s:1240", "s:1267" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1329", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:1222", "inputs": [ "s:1223" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1331", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1223", "inputs": [ "s:37", "s:1224" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1330", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1224", "inputs": [ "s:1225" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1333", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1225", "inputs": [ "s:1226", "s:998", "s:1234" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1332", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:1226", "inputs": [ "s:1227" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1335", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1227", "inputs": [ "s:1228" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "ea2845900b5856c9bf354b1aa9761b5aa6888e5ed61738fe9579ca42bc0f6054", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1336", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175507812499866, "lower_unbounded": false, "upper": 10.504351015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1228", "inputs": [ "s:1229" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1338", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0021755078125, "lower_unbounded": false, "upper": 7.504351015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1229", "inputs": [ "s:1230", "s:1233" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1337", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1230", "inputs": [ "s:1231" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1339", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1231", "inputs": [ "s:1232", "s:1232", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "4a219c06af2467623290b375e580c952ecdae33edf14aa594ac393d27e76b67e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1340", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1232", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1340", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1233", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "ea2845900b5856c9bf354b1aa9761b5aa6888e5ed61738fe9579ca42bc0f6054", "shape": [ 1 ], "values": [ 3.0 ] } ], "region": "r:1341", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 3.0, "lower_unbounded": false, "upper": 3.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1234", "inputs": [ "s:1235", "s:1005", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1342", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1235", "inputs": [ "s:1236" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9f7ce09d183259d69fb622b5248bdb85f5f8493e54529def7b337f69760ed024", "shape": [ 4, 4 ] } ], "region": "r:1344", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1236", "inputs": [ "s:1237" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "f6bb1294da2f78cd935b01c7656280df5eaa0439e9d97bc03775825a41a508e4", "shape": [ 4 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "5998e9d1dd9bf48af6af87a1ef8c7783b818ac8329637ada08cc5fb329042376", "shape": [ 4, 4 ] } ], "region": "r:1345", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1237", "inputs": [ "s:1238" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9017d307c9bd1ac42160d5fae5ff3c4ea7782dd5541b1e83341861e134cae614", "shape": [ 16 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "e4c555906ab7dff40f54eb6916d8ede133b922f31fbcd7f2d25bd787286bf7d3", "shape": [ 16, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e74125e44e9343a83c86395ec81731b98f9aefbe35b61529fe730de3c82a8aab", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e14367196378c02a631dc1b99191297e21c25fac83ce07c8ff35070293c44ccd", "shape": [ 16, 4 ] } ], "region": "r:1347", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 16 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:1238", "inputs": [ "s:1227", "s:1239" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1346", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175507812499866, "lower_unbounded": false, "upper": 11.504351015625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1239", "inputs": [ "s:1227" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1348", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.9978244921875001, "lower_unbounded": false, "upper": 11.504351015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1240", "inputs": [ "s:1241" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1350", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1241", "inputs": [ "s:65", "s:1242" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1349", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1242", "inputs": [ "s:1243" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1352", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1243", "inputs": [ "s:1244", "s:1112", "s:1261" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1351", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:1244", "inputs": [ "s:1245" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1354", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1245", "inputs": [ "s:1246", "s:1253" ], "name": null, "op": "Add", "parameters": [], "region": "r:1356", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0043510117187497, "lower_unbounded": false, "upper": 10.339859852864583, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1246", "inputs": [ "s:1247" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "ea2845900b5856c9bf354b1aa9761b5aa6888e5ed61738fe9579ca42bc0f6054", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1357", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755078124999, "lower_unbounded": false, "upper": 10.337684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1247", "inputs": [ "s:1248" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1359", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.0021755078125, "lower_unbounded": false, "upper": 7.337684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1248", "inputs": [ "s:1249", "s:1252" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1358", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1249", "inputs": [ "s:1250" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1360", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1250", "inputs": [ "s:1251", "s:1251", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "4a219c06af2467623290b375e580c952ecdae33edf14aa594ac393d27e76b67e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1361", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1251", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1361", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1252", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d88c86f15bbea365d658ad95a81d45367c465f7af6f7264fb077f01747ddc77d", "shape": [ 1 ], "values": [ 2.0 ] } ], "region": "r:1362", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 2.0, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1253", "inputs": [ "s:1254" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1363", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755039062499, "lower_unbounded": false, "upper": 0.002175503906249987, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1254", "inputs": [ "s:1255" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:1365", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1255", "inputs": [ "s:1256" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1366", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1256", "inputs": [ "s:1257" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:1369", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1257", "inputs": [ "s:1258" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1371", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1258", "inputs": [ "s:1259" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "52a4c625d7de844b98f1c5c39cad719d87a498955808665a99ea71b5d81e1f93", "shape": [ 33, 1 ] } ], "region": "r:1372", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1259", "inputs": [ "s:1260", "s:1260", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "c570eb9ab41f73b36bf50b3fd1503550ce8d9bc28113f355f45e5df3f5faa9b3", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1373", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1260", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1373", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1261", "inputs": [ "s:1262", "s:1037", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "be94494d5c98aa4e14ad64130eab9a1dc7a79a46a59d9e84d353075468c0a9fd", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "59cc5d44d3cb84eddef86ad62b93e827db0d8a54aa8a8364814b8a098a8c6168", "shape": [ 3, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1374", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1262", "inputs": [ "s:1263" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:1376", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1263", "inputs": [ "s:1264" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:1377", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1264", "inputs": [ "s:1265" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:1379", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:1265", "inputs": [ "s:1245", "s:1266" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1378", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0043510117187497, "lower_unbounded": false, "upper": 11.339859852864583, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1266", "inputs": [ "s:1245" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1380", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510117187497, "lower_unbounded": false, "upper": 11.339859852864583, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1267", "inputs": [ "s:1268" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1382", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1268", "inputs": [ "s:74", "s:1269" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1381", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1269", "inputs": [ "s:1270" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc0a03a8e9c3e933278bf7f2667ac150ebffc7b84c9d541ff71e6243a22d70ca", "shape": [ 7, 33 ] } ], "region": "r:1383", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1270", "inputs": [ "s:1271" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "3addfb141cd7c9c4c6543a82191a3707ac29c7a041217782e61d4d91c691aee8", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "0a2ff5f661b6232dedd8c1bb0416601df78831ad504c44bd61acd771d4cbdd95", "shape": [ 7, 7 ] } ], "region": "r:1385", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1271", "inputs": [ "s:1272" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6f3a1a9d59ae648b4f66e10802905ef54f901e2409b9e01e653a05fd7c4ff786", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "68e014055f97f84bdfabe73267c9ca5a45445840d62edf66ebb0ad398c1d088d", "shape": [ 7, 7 ] } ], "region": "r:1386", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1272", "inputs": [ "s:1273" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ecd290af9b878d500003aada26fc578ffe18fb71ca439df97b2c5da3577bb637", "shape": [ 28 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ac723062162c7c0744a28c7316bd9750d4304fa46e93681ca5ee8499e8dd7cc9", "shape": [ 28, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "adc7fd999de1c34f9ca31a3d93b5f0a159b3c9ff4d6d8ec10bcf843b21f3f38b", "shape": [ 7 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a87893910c503ca82fca06d9c0032304853a7ad9a6b99559819b7f43eb87d121", "shape": [ 28, 7 ] } ], "region": "r:1388", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 28 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1273", "inputs": [ "s:1274", "s:1285" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1387", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1274", "inputs": [ "s:1275", "s:1276" ], "name": null, "op": "Add", "parameters": [], "region": "r:1389", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1275", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1390", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1276", "inputs": [ "s:1277" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:1392", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1277", "inputs": [ "s:1278" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1393", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1278", "inputs": [ "s:1279" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1396", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1279", "inputs": [ "s:1280" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1397", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1280", "inputs": [ "s:1281", "s:1282" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1397", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1281", "inputs": [ "s:1275" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1399", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1282", "inputs": [ "s:1283" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:1401", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1283", "inputs": [ "s:1284", "s:1284", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1402", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1284", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1402", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1285", "inputs": [ "s:1274" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1403", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1286", "inputs": [ "s:1287" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1405", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1287", "inputs": [ "s:1288", "s:1294" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1404", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1288", "inputs": [ "s:1289" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1408", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1289", "inputs": [ "s:1290" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1409", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1290", "inputs": [ "s:1291", "s:1292" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1409", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1291", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "79917cb2415012d945ab2169ee0312f15acd06846b035207da169eb0ce402674", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1411", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1292", "inputs": [ "s:1293" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c685308f2ab536a2889ee91d69f78db27e7708c0ae08ee9452a5f58d5395e79b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1413", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1293", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1414", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1294", "inputs": [ "s:1295" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1416", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1295", "inputs": [ "s:1296", "s:1299", "s:1302" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1416", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:1296", "inputs": [ "s:1297" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1418", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1297", "inputs": [ "s:37", "s:1298" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1417", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1298", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1419", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1299", "inputs": [ "s:1300" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1421", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1300", "inputs": [ "s:65", "s:1301" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1420", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1301", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1422", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1302", "inputs": [ "s:1303" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1424", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1303", "inputs": [ "s:74", "s:1304" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1423", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1304", "inputs": [ "s:1305" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fc0a03a8e9c3e933278bf7f2667ac150ebffc7b84c9d541ff71e6243a22d70ca", "shape": [ 7, 33 ] } ], "region": "r:1425", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1305", "inputs": [ "s:1306" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "3addfb141cd7c9c4c6543a82191a3707ac29c7a041217782e61d4d91c691aee8", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "0a2ff5f661b6232dedd8c1bb0416601df78831ad504c44bd61acd771d4cbdd95", "shape": [ 7, 7 ] } ], "region": "r:1427", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1306", "inputs": [ "s:1307" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "6f3a1a9d59ae648b4f66e10802905ef54f901e2409b9e01e653a05fd7c4ff786", "shape": [ 7 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "68e014055f97f84bdfabe73267c9ca5a45445840d62edf66ebb0ad398c1d088d", "shape": [ 7, 7 ] } ], "region": "r:1428", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1307", "inputs": [ "s:1308" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ecd290af9b878d500003aada26fc578ffe18fb71ca439df97b2c5da3577bb637", "shape": [ 28 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ac723062162c7c0744a28c7316bd9750d4304fa46e93681ca5ee8499e8dd7cc9", "shape": [ 28, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "adc7fd999de1c34f9ca31a3d93b5f0a159b3c9ff4d6d8ec10bcf843b21f3f38b", "shape": [ 7 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a87893910c503ca82fca06d9c0032304853a7ad9a6b99559819b7f43eb87d121", "shape": [ 28, 7 ] } ], "region": "r:1430", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 28 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 7 }, { "annotation": null, "id": "s:1308", "inputs": [ "s:1309", "s:1320" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1429", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1309", "inputs": [ "s:1310", "s:1311" ], "name": null, "op": "Add", "parameters": [], "region": "r:1431", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "integer-valued" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 8.0, "upper_unbounded": false }, "integer": true, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 7.002175503906249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1310", "inputs": [ "s:8" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1432", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1311", "inputs": [ "s:1312" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:1434", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1312", "inputs": [ "s:1313" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1435", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1313", "inputs": [ "s:1314" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1438", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1314", "inputs": [ "s:1315" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1439", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1315", "inputs": [ "s:1316", "s:1317" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1439", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1316", "inputs": [ "s:1310" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "db8d1a556ec9527f63930661298f4b8607cfcecb3c364235751acfef1ec8ed1a", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1441", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1317", "inputs": [ "s:1318" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5e4721092354da41fce47e2699eedd57e17f0c856f666a6c788018ae740d0d77", "shape": [ 1, 33 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:1443", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1318", "inputs": [ "s:1319", "s:1319", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1444", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1319", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1444", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1320", "inputs": [ "s:1309" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1445", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.99782449609375, "lower_unbounded": false, "upper": 8.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1321", "inputs": [ "s:1322" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1447", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=2.01))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1322", "inputs": [ "s:1323", "s:1329" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1446", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1323", "inputs": [ "s:1324" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1450", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1324", "inputs": [ "s:1325" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1451", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1325", "inputs": [ "s:1326", "s:1327" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1451", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1326", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "06880e085843c937418cc66e843c4da6c39a4617eaea1521f400b8f849e67dfa", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1453", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1327", "inputs": [ "s:1328" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "78ce7ad60acf4307970c9a59f23a0d670b828dd102abe6c47a1ed75fbbb679f6", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1455", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1328", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1456", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1329", "inputs": [ "s:1330" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1458", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 2.01, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1330", "inputs": [ "s:1331", "s:1334", "s:1337" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1458", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:1331", "inputs": [ "s:1332" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1460", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1332", "inputs": [ "s:37", "s:1333" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1459", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1333", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1461", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1334", "inputs": [ "s:1335" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1463", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1335", "inputs": [ "s:65", "s:1336" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1462", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1336", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1464", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1337", "inputs": [ "s:1338" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1466", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1338", "inputs": [ "s:74", "s:1339" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1465", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1339", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "d3d4b4498b152d62ffae25a2127656848110f4397c0385d2fa75457e20fbbf30", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "region": "r:1467", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1340", "inputs": [ "s:1341" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1469", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0200249999999995))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1341", "inputs": [ "s:1342", "s:1348" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1468", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1342", "inputs": [ "s:1343" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1472", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1343", "inputs": [ "s:1344" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1473", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1344", "inputs": [ "s:1345", "s:1346" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1473", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1345", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0d35a65ae9d33e342992c713cdfe6e51a88a3ccca57c25d7003ecaef1a060be9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1475", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1346", "inputs": [ "s:1347" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5eab47f016015070cba2c5ceb2d8bececfdf333d0f25fc7ab6c73cb8201cf7cc", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1477", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1347", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1478", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1348", "inputs": [ "s:1349" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1480", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1349", "inputs": [ "s:1350", "s:1353", "s:1356" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1480", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:1350", "inputs": [ "s:1351" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1482", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1351", "inputs": [ "s:37", "s:1352" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1481", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1352", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1483", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1353", "inputs": [ "s:1354" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1485", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1354", "inputs": [ "s:65", "s:1355" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1484", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1355", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1486", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1356", "inputs": [ "s:1357" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1488", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1357", "inputs": [ "s:74", "s:1358" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1487", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1358", "inputs": [ "s:1359" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1490", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1359", "inputs": [ "s:1360", "s:1368", "s:1369" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1489", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:1360", "inputs": [ "s:1361" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b8bb7a27e55e02d67b43582292d3134e5715248dc3f4a54dac7e1c71519551b5", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1492", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1361", "inputs": [ "s:1362" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1493", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755078125, "lower_unbounded": false, "upper": 6.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1362", "inputs": [ "s:1363" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1495", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755078125, "lower_unbounded": false, "upper": 6.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1363", "inputs": [ "s:1364", "s:1367" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1494", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1364", "inputs": [ "s:1365" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1496", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1365", "inputs": [ "s:1366", "s:1366", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "10a7f03eb2fc77dedb404781bd8e8a871cccf1430e4c988c709e4a72fc31684e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1497", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1366", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1497", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1367", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "fca31f1667a6aa1bba12fca4e4ea1becd503379d80da3213af07f6cc5702828d", "shape": [ 1 ], "values": [ 5.0 ] } ], "region": "r:1498", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 5.0, "lower_unbounded": false, "upper": 5.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1368", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1499", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1369", "inputs": [ "s:1370", "s:1375", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "0ebfee506020ed8b5565a2f5252bea8bae295626330b537d4c9b488a46a712de", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "9010fceb49e61304a784d8dc3d60e4881a4749caa665c18a91caecfad9bbce83", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1500", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1370", "inputs": [ "s:1371" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "9d908ecfb6b256def8b49a7c504e6c889c4b0e41fe6ce3e01863dd7b61a20aa0", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f1af9fe3a978a5ad01515b76da2145665eca0ffdd72ab3b8e526c05b1dd8d2f5", "shape": [ 6, 6 ] } ], "region": "r:1502", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1371", "inputs": [ "s:1372" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "f46913286d895f62d646314f400fc3d7a543cf34eb4ae56df87f43dc4582a36b", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "bbb822a1b4ef657e9edb94455836b2a171d5c4ac15e687630dabed0b646c4421", "shape": [ 6, 6 ] } ], "region": "r:1503", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1372", "inputs": [ "s:1373" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1fa343150bb83ed9689c44694dc284a239460221edd24178bbe9c0642874e4cf", "shape": [ 24 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "38c349087c69fb3195cc6f14b0efaee2cbddf46673bcf566cbd455c6fc27569f", "shape": [ 24, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "90ba88e2e295f2a5205122dcf0b0d15b5e9bd0ee828ef8e3e4ddf3c6bc53a320", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "062c0d53503ab27e308d67f283dcb28c04de7a7d7e721bf62691c3f373150efb", "shape": [ 24, 6 ] } ], "region": "r:1505", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 24 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1373", "inputs": [ "s:1361", "s:1374" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1504", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755078125, "lower_unbounded": false, "upper": 7.837684348958334, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1374", "inputs": [ "s:1361" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1506", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175507812500088, "lower_unbounded": false, "upper": 7.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1375", "inputs": [ "s:1376" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "9d908ecfb6b256def8b49a7c504e6c889c4b0e41fe6ce3e01863dd7b61a20aa0", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f1af9fe3a978a5ad01515b76da2145665eca0ffdd72ab3b8e526c05b1dd8d2f5", "shape": [ 6, 6 ] } ], "region": "r:1508", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1376", "inputs": [ "s:1377" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "f46913286d895f62d646314f400fc3d7a543cf34eb4ae56df87f43dc4582a36b", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "bbb822a1b4ef657e9edb94455836b2a171d5c4ac15e687630dabed0b646c4421", "shape": [ 6, 6 ] } ], "region": "r:1509", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1377", "inputs": [ "s:1378" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1fa343150bb83ed9689c44694dc284a239460221edd24178bbe9c0642874e4cf", "shape": [ 24 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "38c349087c69fb3195cc6f14b0efaee2cbddf46673bcf566cbd455c6fc27569f", "shape": [ 24, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "90ba88e2e295f2a5205122dcf0b0d15b5e9bd0ee828ef8e3e4ddf3c6bc53a320", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "062c0d53503ab27e308d67f283dcb28c04de7a7d7e721bf62691c3f373150efb", "shape": [ 24, 6 ] } ], "region": "r:1511", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 24 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1378", "inputs": [ "s:1379", "s:1380" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1510", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -8.357438532905364, "lower_unbounded": false, "upper": 22.52410519957203, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1379", "inputs": [ "s:18" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "053ac09e0109c6d84088a0106d301d00ebe129455cb7889c911000d6f462271f", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1512", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -8.357438532905364, "lower_unbounded": false, "upper": 21.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1380", "inputs": [ "s:1379" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1513", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -7.357438532905364, "lower_unbounded": false, "upper": 22.52410519957203, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1381", "inputs": [ "s:1382" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1515", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0200249999999995))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1382", "inputs": [ "s:1383", "s:1389" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1514", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1383", "inputs": [ "s:1384" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1518", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1384", "inputs": [ "s:1385" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1519", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1385", "inputs": [ "s:1386", "s:1387" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1519", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1386", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "d484ed60a29fdabc7a9e604d97a1de9b86fb26900f9f27e08f9e560d463c68b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1521", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1387", "inputs": [ "s:1388" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6537e8db32fb220fff8c4e77ee94d0ed69757893ff2f79d6442ed507ed42042c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1523", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1388", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1524", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1389", "inputs": [ "s:1390" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1526", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1390", "inputs": [ "s:1391", "s:1394", "s:1397" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1526", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:1391", "inputs": [ "s:1392" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1528", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1392", "inputs": [ "s:37", "s:1393" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1527", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1393", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1529", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1394", "inputs": [ "s:1395" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1531", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1395", "inputs": [ "s:65", "s:1396" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1530", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1396", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1532", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1397", "inputs": [ "s:1398" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1534", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1398", "inputs": [ "s:74", "s:1399" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1533", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1399", "inputs": [ "s:1400" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1536", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1400", "inputs": [ "s:1401", "s:1368", "s:1409" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1535", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:1401", "inputs": [ "s:1402" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b8bb7a27e55e02d67b43582292d3134e5715248dc3f4a54dac7e1c71519551b5", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1538", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1402", "inputs": [ "s:1403" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1539", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175507812500088, "lower_unbounded": false, "upper": 7.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1403", "inputs": [ "s:1404" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1541", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755078125, "lower_unbounded": false, "upper": 6.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1404", "inputs": [ "s:1405", "s:1408" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1540", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1405", "inputs": [ "s:1406" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1542", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1406", "inputs": [ "s:1407", "s:1407", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "2169e1e94e1e1b2d98e429e6ce8e03441aba029c2b9ac34cc044c8298ec48659", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1543", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1407", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1543", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1408", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "fca31f1667a6aa1bba12fca4e4ea1becd503379d80da3213af07f6cc5702828d", "shape": [ 1 ], "values": [ 5.0 ] } ], "region": "r:1544", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 5.0, "lower_unbounded": false, "upper": 5.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1409", "inputs": [ "s:1410", "s:1375", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "0ebfee506020ed8b5565a2f5252bea8bae295626330b537d4c9b488a46a712de", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "9010fceb49e61304a784d8dc3d60e4881a4749caa665c18a91caecfad9bbce83", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1545", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1410", "inputs": [ "s:1411" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "9d908ecfb6b256def8b49a7c504e6c889c4b0e41fe6ce3e01863dd7b61a20aa0", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f1af9fe3a978a5ad01515b76da2145665eca0ffdd72ab3b8e526c05b1dd8d2f5", "shape": [ 6, 6 ] } ], "region": "r:1547", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1411", "inputs": [ "s:1412" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "f46913286d895f62d646314f400fc3d7a543cf34eb4ae56df87f43dc4582a36b", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "bbb822a1b4ef657e9edb94455836b2a171d5c4ac15e687630dabed0b646c4421", "shape": [ 6, 6 ] } ], "region": "r:1548", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1412", "inputs": [ "s:1413" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1fa343150bb83ed9689c44694dc284a239460221edd24178bbe9c0642874e4cf", "shape": [ 24 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "38c349087c69fb3195cc6f14b0efaee2cbddf46673bcf566cbd455c6fc27569f", "shape": [ 24, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "90ba88e2e295f2a5205122dcf0b0d15b5e9bd0ee828ef8e3e4ddf3c6bc53a320", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "062c0d53503ab27e308d67f283dcb28c04de7a7d7e721bf62691c3f373150efb", "shape": [ 24, 6 ] } ], "region": "r:1550", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 24 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1413", "inputs": [ "s:1402", "s:1414" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1549", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175507812500088, "lower_unbounded": false, "upper": 8.837684348958334, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1414", "inputs": [ "s:1402" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1551", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.9978244921874999, "lower_unbounded": false, "upper": 8.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1415", "inputs": [ "s:1416" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1553", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0200249999999995))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1416", "inputs": [ "s:1417", "s:1423" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1552", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1417", "inputs": [ "s:1418" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1556", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1418", "inputs": [ "s:1419" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1557", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1419", "inputs": [ "s:1420", "s:1421" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1557", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1420", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5b5d5ecc535fb6f3f2089b1ebe1411a7aee7a3b83c453733fd6335b0fa0cdbc7", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1559", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1421", "inputs": [ "s:1422" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "cb24e6346737ac5db8a9f0fd7ac795a0a2dcdb2d3606e790469a23c68df90eab", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1561", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1422", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1562", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1423", "inputs": [ "s:1424" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1564", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1424", "inputs": [ "s:1425", "s:1428", "s:1431" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1564", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:1425", "inputs": [ "s:1426" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1566", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1426", "inputs": [ "s:37", "s:1427" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1565", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1427", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1567", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1428", "inputs": [ "s:1429" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1569", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1429", "inputs": [ "s:65", "s:1430" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1568", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1430", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1570", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1431", "inputs": [ "s:1432" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1572", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1432", "inputs": [ "s:74", "s:1433" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1571", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1433", "inputs": [ "s:1434" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1574", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1434", "inputs": [ "s:1435", "s:1368", "s:1443" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1573", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:1435", "inputs": [ "s:1436" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b8bb7a27e55e02d67b43582292d3134e5715248dc3f4a54dac7e1c71519551b5", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1576", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1436", "inputs": [ "s:1437" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d88c86f15bbea365d658ad95a81d45367c465f7af6f7264fb077f01747ddc77d", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1577", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.9978244921874999, "lower_unbounded": false, "upper": 8.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1437", "inputs": [ "s:1438" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1579", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755078125, "lower_unbounded": false, "upper": 6.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1438", "inputs": [ "s:1439", "s:1442" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1578", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1439", "inputs": [ "s:1440" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1580", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1440", "inputs": [ "s:1441", "s:1441", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "a553a0a36d2e36a7f6e98cf3ed0070972bb5a4159a576ded3b4dabda7861042d", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1581", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1441", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1581", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1442", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "fca31f1667a6aa1bba12fca4e4ea1becd503379d80da3213af07f6cc5702828d", "shape": [ 1 ], "values": [ 5.0 ] } ], "region": "r:1582", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 5.0, "lower_unbounded": false, "upper": 5.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1443", "inputs": [ "s:1444", "s:1375", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "0ebfee506020ed8b5565a2f5252bea8bae295626330b537d4c9b488a46a712de", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "9010fceb49e61304a784d8dc3d60e4881a4749caa665c18a91caecfad9bbce83", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1583", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1444", "inputs": [ "s:1445" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "9d908ecfb6b256def8b49a7c504e6c889c4b0e41fe6ce3e01863dd7b61a20aa0", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f1af9fe3a978a5ad01515b76da2145665eca0ffdd72ab3b8e526c05b1dd8d2f5", "shape": [ 6, 6 ] } ], "region": "r:1585", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1445", "inputs": [ "s:1446" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "f46913286d895f62d646314f400fc3d7a543cf34eb4ae56df87f43dc4582a36b", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "bbb822a1b4ef657e9edb94455836b2a171d5c4ac15e687630dabed0b646c4421", "shape": [ 6, 6 ] } ], "region": "r:1586", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1446", "inputs": [ "s:1447" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1fa343150bb83ed9689c44694dc284a239460221edd24178bbe9c0642874e4cf", "shape": [ 24 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "38c349087c69fb3195cc6f14b0efaee2cbddf46673bcf566cbd455c6fc27569f", "shape": [ 24, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "90ba88e2e295f2a5205122dcf0b0d15b5e9bd0ee828ef8e3e4ddf3c6bc53a320", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "062c0d53503ab27e308d67f283dcb28c04de7a7d7e721bf62691c3f373150efb", "shape": [ 24, 6 ] } ], "region": "r:1588", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 24 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1447", "inputs": [ "s:1436", "s:1448" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1587", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.9978244921874999, "lower_unbounded": false, "upper": 9.837684348958334, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1448", "inputs": [ "s:1436" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1589", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.9978244921875, "lower_unbounded": false, "upper": 9.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1449", "inputs": [ "s:1450" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1591", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0200249999999995))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1450", "inputs": [ "s:1451", "s:1457" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1590", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1451", "inputs": [ "s:1452" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1594", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1452", "inputs": [ "s:1453" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1595", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1453", "inputs": [ "s:1454", "s:1455" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1595", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1454", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "55c549ac2b96d8954127159a714c5c16d1c42d0fa07f2b1c558bbe5d5fae3108", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1597", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1455", "inputs": [ "s:1456" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c359380011b0a1c803214db4688b883faec47605ccd8371302b87b11efd33ee3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1599", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1456", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1600", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1457", "inputs": [ "s:1458" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1602", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1458", "inputs": [ "s:1459", "s:1462", "s:1465" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1602", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:1459", "inputs": [ "s:1460" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1604", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1460", "inputs": [ "s:37", "s:1461" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1603", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1461", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1605", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1462", "inputs": [ "s:1463" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1607", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1463", "inputs": [ "s:65", "s:1464" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1606", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1464", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1608", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1465", "inputs": [ "s:1466" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1610", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1466", "inputs": [ "s:74", "s:1467" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1609", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1467", "inputs": [ "s:1468" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1612", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1468", "inputs": [ "s:1469", "s:1368", "s:1477" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1611", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:1469", "inputs": [ "s:1470" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b8bb7a27e55e02d67b43582292d3134e5715248dc3f4a54dac7e1c71519551b5", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1614", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1470", "inputs": [ "s:1471" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "ea2845900b5856c9bf354b1aa9761b5aa6888e5ed61738fe9579ca42bc0f6054", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1615", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.9978244921875, "lower_unbounded": false, "upper": 9.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1471", "inputs": [ "s:1472" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1617", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755078125, "lower_unbounded": false, "upper": 6.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1472", "inputs": [ "s:1473", "s:1476" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1616", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1473", "inputs": [ "s:1474" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1618", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1474", "inputs": [ "s:1475", "s:1475", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "4a219c06af2467623290b375e580c952ecdae33edf14aa594ac393d27e76b67e", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1619", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1475", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1619", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1476", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "fca31f1667a6aa1bba12fca4e4ea1becd503379d80da3213af07f6cc5702828d", "shape": [ 1 ], "values": [ 5.0 ] } ], "region": "r:1620", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 5.0, "lower_unbounded": false, "upper": 5.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1477", "inputs": [ "s:1478", "s:1375", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "0ebfee506020ed8b5565a2f5252bea8bae295626330b537d4c9b488a46a712de", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "9010fceb49e61304a784d8dc3d60e4881a4749caa665c18a91caecfad9bbce83", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1621", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1478", "inputs": [ "s:1479" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "9d908ecfb6b256def8b49a7c504e6c889c4b0e41fe6ce3e01863dd7b61a20aa0", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f1af9fe3a978a5ad01515b76da2145665eca0ffdd72ab3b8e526c05b1dd8d2f5", "shape": [ 6, 6 ] } ], "region": "r:1623", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1479", "inputs": [ "s:1480" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "f46913286d895f62d646314f400fc3d7a543cf34eb4ae56df87f43dc4582a36b", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "bbb822a1b4ef657e9edb94455836b2a171d5c4ac15e687630dabed0b646c4421", "shape": [ 6, 6 ] } ], "region": "r:1624", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1480", "inputs": [ "s:1481" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1fa343150bb83ed9689c44694dc284a239460221edd24178bbe9c0642874e4cf", "shape": [ 24 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "38c349087c69fb3195cc6f14b0efaee2cbddf46673bcf566cbd455c6fc27569f", "shape": [ 24, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "90ba88e2e295f2a5205122dcf0b0d15b5e9bd0ee828ef8e3e4ddf3c6bc53a320", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "062c0d53503ab27e308d67f283dcb28c04de7a7d7e721bf62691c3f373150efb", "shape": [ 24, 6 ] } ], "region": "r:1626", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 24 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1481", "inputs": [ "s:1470", "s:1482" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1625", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.9978244921875, "lower_unbounded": false, "upper": 10.837684348958334, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1482", "inputs": [ "s:1470" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1627", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 2.9978244921875, "lower_unbounded": false, "upper": 10.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1483", "inputs": [ "s:1484" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1629", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0200249999999995))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1484", "inputs": [ "s:1485", "s:1491" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1628", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1485", "inputs": [ "s:1486" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1632", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1486", "inputs": [ "s:1487" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1633", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1487", "inputs": [ "s:1488", "s:1489" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1633", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1488", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "aa35d7d5d29f2463284c11f8021afed217eca0782d8286f903a06fa4b8e5fe00", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1635", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1489", "inputs": [ "s:1490" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c6f408755f3e0820541405e08e18c2b65515c3a800b5113c379c2596aadbe210", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1637", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1490", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1638", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1491", "inputs": [ "s:1492" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1640", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1492", "inputs": [ "s:1493", "s:1496", "s:1499" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1640", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:1493", "inputs": [ "s:1494" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1642", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1494", "inputs": [ "s:37", "s:1495" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1641", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1495", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1643", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1496", "inputs": [ "s:1497" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1645", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1497", "inputs": [ "s:65", "s:1498" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1644", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1498", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1646", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1499", "inputs": [ "s:1500" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1648", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1500", "inputs": [ "s:74", "s:1501" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1647", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1501", "inputs": [ "s:1502" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1650", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1502", "inputs": [ "s:1503", "s:1368", "s:1511" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1649", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:1503", "inputs": [ "s:1504" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b8bb7a27e55e02d67b43582292d3134e5715248dc3f4a54dac7e1c71519551b5", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1652", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1504", "inputs": [ "s:1505" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "4f4b9b7d8b86633e2824e2f439819357b0cd010ab410ea1a691b12c5f94e91e0", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1653", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 2.9978244921875, "lower_unbounded": false, "upper": 10.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1505", "inputs": [ "s:1506" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1655", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755078125, "lower_unbounded": false, "upper": 6.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1506", "inputs": [ "s:1507", "s:1510" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1654", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1507", "inputs": [ "s:1508" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1656", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1508", "inputs": [ "s:1509", "s:1509", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5e0b5d8b5834e12ad26a163a18e4f7f02a3faaf6425a8e8dd6883adc4c4db7a6", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1657", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1509", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1657", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1510", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "fca31f1667a6aa1bba12fca4e4ea1becd503379d80da3213af07f6cc5702828d", "shape": [ 1 ], "values": [ 5.0 ] } ], "region": "r:1658", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 5.0, "lower_unbounded": false, "upper": 5.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1511", "inputs": [ "s:1512", "s:1375", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "0ebfee506020ed8b5565a2f5252bea8bae295626330b537d4c9b488a46a712de", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "9010fceb49e61304a784d8dc3d60e4881a4749caa665c18a91caecfad9bbce83", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1659", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1512", "inputs": [ "s:1513" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "9d908ecfb6b256def8b49a7c504e6c889c4b0e41fe6ce3e01863dd7b61a20aa0", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f1af9fe3a978a5ad01515b76da2145665eca0ffdd72ab3b8e526c05b1dd8d2f5", "shape": [ 6, 6 ] } ], "region": "r:1661", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1513", "inputs": [ "s:1514" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "f46913286d895f62d646314f400fc3d7a543cf34eb4ae56df87f43dc4582a36b", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "bbb822a1b4ef657e9edb94455836b2a171d5c4ac15e687630dabed0b646c4421", "shape": [ 6, 6 ] } ], "region": "r:1662", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1514", "inputs": [ "s:1515" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1fa343150bb83ed9689c44694dc284a239460221edd24178bbe9c0642874e4cf", "shape": [ 24 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "38c349087c69fb3195cc6f14b0efaee2cbddf46673bcf566cbd455c6fc27569f", "shape": [ 24, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "90ba88e2e295f2a5205122dcf0b0d15b5e9bd0ee828ef8e3e4ddf3c6bc53a320", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "062c0d53503ab27e308d67f283dcb28c04de7a7d7e721bf62691c3f373150efb", "shape": [ 24, 6 ] } ], "region": "r:1664", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 24 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1515", "inputs": [ "s:1504", "s:1516" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1663", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 2.9978244921875, "lower_unbounded": false, "upper": 11.837684348958334, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1516", "inputs": [ "s:1504" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1665", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 3.9978244921875, "lower_unbounded": false, "upper": 11.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1517", "inputs": [ "s:1518" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1667", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0200249999999995))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1518", "inputs": [ "s:1519", "s:1525" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1666", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1519", "inputs": [ "s:1520" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1670", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1520", "inputs": [ "s:1521" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1671", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1521", "inputs": [ "s:1522", "s:1523" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1671", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1522", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "dee95cfd95fe9c0151dba42d63cf451b06d57f8e26bec9cde7ae4ccc858772aa", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1673", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1523", "inputs": [ "s:1524" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c139a06e54e1e0f0339c3619fdcd664073a35456fe9886d9798195e0e0379efb", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1675", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1524", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1676", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1525", "inputs": [ "s:1526" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1678", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0200249999999995, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1526", "inputs": [ "s:1527", "s:1530", "s:1533" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1678", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0100249999999997, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:1527", "inputs": [ "s:1528" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1680", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1528", "inputs": [ "s:37", "s:1529" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1679", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1529", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1681", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1530", "inputs": [ "s:1531" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1683", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1531", "inputs": [ "s:65", "s:1532" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1682", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1532", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1684", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1533", "inputs": [ "s:1534" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1686", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1534", "inputs": [ "s:74", "s:1535" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1685", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1535", "inputs": [ "s:1536" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "afe083a6cb3a0f63bb2e9450eed3cd85de06a6bede424bbd192a8da3e950e001", "shape": [ 66, 67 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "6392b1c348b80e614e84fcd818cac7ecbc12861263f40e33d611e4f8ff145b97", "shape": [ 66, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "44b8aa4d28701168922acf61435ea4bb442f97b0b14ad7a2510ed68874ee2a72", "shape": [ 66 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "b919838ca8bcdb600eb5d3a979ea26f1447c675ccd45afd6400664a519625093", "shape": [ 66, 67 ] } ], "region": "r:1688", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 66 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1536", "inputs": [ "s:1537", "s:1368", "s:1545" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1687", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 67 }, { "annotation": null, "id": "s:1537", "inputs": [ "s:1538" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b8bb7a27e55e02d67b43582292d3134e5715248dc3f4a54dac7e1c71519551b5", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1690", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1538", "inputs": [ "s:1539" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "fca31f1667a6aa1bba12fca4e4ea1becd503379d80da3213af07f6cc5702828d", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1691", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 3.9978244921875, "lower_unbounded": false, "upper": 11.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1539", "inputs": [ "s:1540" ], "name": "min", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a8d32a0a8c6571359094a5e800a1b0ce79202e914459f121231d0095312bfebc", "shape": [ 3, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "948cb73bffff669c19fd09560cae71a10079e03e09ae5b5ba488a6e4dd48d80f", "shape": [ 3, 1 ] } ], "region": "r:1693", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 3 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0021755078125, "lower_unbounded": false, "upper": 6.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1540", "inputs": [ "s:1541", "s:1544" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1692", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1541", "inputs": [ "s:1542" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "40b182dfdf652337836c79068dd80149e637f719d44beaf44b21218b98eadc23", "shape": [ 33, 1 ] } ], "region": "r:1694", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=6.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 6.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1542", "inputs": [ "s:1543", "s:1543", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "42861c9444db42b4d1d4965d39b2485c070ce6152a23485353d3514c029140fc", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "297da7e89fe2e9a43ea48ec6d4ba366e6b697c163a2dfe638d833058abc79596", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1695", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1543", "inputs": [], "name": "rotary_offset_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:1695", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1544", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "fca31f1667a6aa1bba12fca4e4ea1becd503379d80da3213af07f6cc5702828d", "shape": [ 1 ], "values": [ 5.0 ] } ], "region": "r:1696", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 5.0, "lower_unbounded": false, "upper": 5.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1545", "inputs": [ "s:1546", "s:1375", "s:8" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "0ebfee506020ed8b5565a2f5252bea8bae295626330b537d4c9b488a46a712de", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "9010fceb49e61304a784d8dc3d60e4881a4749caa665c18a91caecfad9bbce83", "shape": [ 6, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "7a5bb7be9672edff18cdf8744a3b5a67eebd12427afa9f0bb5d9fcd0838b7cc4", "shape": [ 33, 33 ] } ], "region": "r:1697", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 33, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1546", "inputs": [ "s:1547" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "9d908ecfb6b256def8b49a7c504e6c889c4b0e41fe6ce3e01863dd7b61a20aa0", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f1af9fe3a978a5ad01515b76da2145665eca0ffdd72ab3b8e526c05b1dd8d2f5", "shape": [ 6, 6 ] } ], "region": "r:1699", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1547", "inputs": [ "s:1548" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "f46913286d895f62d646314f400fc3d7a543cf34eb4ae56df87f43dc4582a36b", "shape": [ 6 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "bbb822a1b4ef657e9edb94455836b2a171d5c4ac15e687630dabed0b646c4421", "shape": [ 6, 6 ] } ], "region": "r:1700", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1548", "inputs": [ "s:1549" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1fa343150bb83ed9689c44694dc284a239460221edd24178bbe9c0642874e4cf", "shape": [ 24 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "38c349087c69fb3195cc6f14b0efaee2cbddf46673bcf566cbd455c6fc27569f", "shape": [ 24, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "90ba88e2e295f2a5205122dcf0b0d15b5e9bd0ee828ef8e3e4ddf3c6bc53a320", "shape": [ 6 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "062c0d53503ab27e308d67f283dcb28c04de7a7d7e721bf62691c3f373150efb", "shape": [ 24, 6 ] } ], "region": "r:1702", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 24 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:1549", "inputs": [ "s:1538", "s:1550" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1701", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 3.9978244921875, "lower_unbounded": false, "upper": 12.837684348958334, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1550", "inputs": [ "s:1538" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:1703", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 4.9978244921875, "lower_unbounded": false, "upper": 12.837684348958334, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1551", "inputs": [ "s:1552" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1705", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1552", "inputs": [ "s:1553", "s:1559" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1704", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1553", "inputs": [ "s:1554" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1708", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1554", "inputs": [ "s:1555" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:1709", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1555", "inputs": [ "s:1556", "s:1557" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1709", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:1556", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "bbc16069d33e5cd6fc8fbe89aba5cbcd6379e2926aba86ed299b10c60134689d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1711", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1557", "inputs": [ "s:1558" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "7a1aad3244914967fc4e6a98bfa0537fa58ab41acab95f80be84f5e2a20bf0ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:1713", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1558", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:1714", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -29.52410519957203, "lower_unbounded": false, "upper": 0.35743853290536454, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:1559", "inputs": [ "s:1560" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9b17f6c31a009fd82e9977776e95baa89cfff4a26317e20416dfc7c14ce2c11b", "shape": [ 99, 33 ] } ], "region": "r:1716", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1560", "inputs": [ "s:1561", "s:1564", "s:1567" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1716", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 99 }, { "annotation": null, "id": "s:1561", "inputs": [ "s:1562" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1718", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1562", "inputs": [ "s:37", "s:1563" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1717", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1563", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1719", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1564", "inputs": [ "s:1565" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1721", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1565", "inputs": [ "s:65", "s:1566" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1720", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1566", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1722", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1567", "inputs": [ "s:1568" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "90f253c8c87f43544750806f6562ba505af0ae44b21e97c2f96b304924c2c4cd", "shape": [ 33, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "00229bb1b80b07faeb8a024ca7566ddb41cb80efbb12735b5b7e5675fb37fe6f", "shape": [ 33, 33 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "115bad14f1c9f2c027a84de21b107015722cb76be8d0abf3760ad8e00d6c24a5", "shape": [ 33 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "956ebdcb6da5f24b34f0aff568e1528f5de6e08de608f5ad8ab1721b51b815bb", "shape": [ 33, 34 ] } ], "region": "r:1724", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 33 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1568", "inputs": [ "s:74", "s:1569" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1723", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:1569", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "6cf235e9ed99ffb952440c66003e1a351befe8c80539e5bd9adffb66ec57747a", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1725", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 }, { "annotation": null, "id": "s:1570", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "219463451d29bbc5e3938d0f5394ec9a438fa2930a11e18f82abcb8091cc15f2", "shape": [ 33 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:1726", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 33 } ], "root": "s:0", "semantic_regions": [ { "annotation": null, "id": "r:0", "op": "swiglu.map_select.select", "operands": [ "s:2", "s:10", "s:1570" ], "params": {}, "parent": null, "results": [ "s:0" ] }, { "annotation": null, "id": "r:1", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:0", "results": [ "s:0" ] }, { "annotation": null, "id": "r:2", "op": "attention_ops.get_prev_value", "operands": [ "s:7", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:2" ] }, { "annotation": null, "id": "r:3", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:7" ] }, { "annotation": null, "id": "r:4", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:3", "results": [ "s:7" ] }, { "annotation": null, "id": "r:5", "op": "inout_nodes.create_onehot_embedding", "operands": [], "params": { "vocab": [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "+", "-", "*", "\n", " ", "", "", "", "\u2070 ", "\u00b9 ", "\u00b2 ", "\u00b3 ", "\u2074 ", "\u2075 ", "\u2076 ", "\u2080 ", "\u2081 ", "\u2082 ", "\u2083 ", "\u2084 ", "\u2085 ", "\u2086 ", "" ] }, "parent": null, "results": [ "s:8" ] }, { "annotation": null, "id": "r:6", "op": "linear.sum_nodes", "operands": [ "s:12", "s:81", "s:277", "s:356", "s:405", "s:467", "s:529", "s:586", "s:644", "s:700", "s:763", "s:823", "s:883", "s:941", "s:976", "s:1063", "s:1138", "s:1212", "s:1286", "s:1321", "s:1340", "s:1381", "s:1415", "s:1449", "s:1483", "s:1517", "s:1551" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:10" ] }, { "annotation": null, "id": "r:7", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:14", "s:33" ], "params": {}, "parent": null, "results": [ "s:12" ] }, { "annotation": null, "id": "r:8", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:13" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:7", "results": [ "s:12" ] }, { "annotation": null, "id": "r:9", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:17", "s:31" ], "params": {}, "parent": null, "results": [ "s:14" ] }, { "annotation": null, "id": "r:10", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:15" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:9", "results": [ "s:14" ] }, { "annotation": null, "id": "r:11", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:15" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:10", "results": [ "s:14" ] }, { "annotation": null, "id": "r:12", "op": "linear.sum_nodes", "operands": [ "s:17", "s:31" ], "params": { "max_fanout": null }, "parent": "r:9", "results": [ "s:15" ] }, { "annotation": null, "id": "r:13", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:17" ] }, { "annotation": null, "id": "r:14", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:13", "results": [ "s:17" ] }, { "annotation": null, "id": "r:15", "op": "swiglu.marker_count.count_since_marker", "operands": [ "s:22", "s:29" ], "params": { "max_gap": 29, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:18" ] }, { "annotation": null, "id": "r:16", "op": "linear.add_const", "operands": [ "s:19" ], "params": { "scalar": -1.0 }, "parent": "r:15", "results": [ "s:18" ] }, { "annotation": null, "id": "r:17", "op": "swiglu.arithmetic_ops.reciprocal", "operands": [ "s:20" ], "params": { "max_value": 1.5, "min_value": 0.03278688524590164, "step": 0.03334575260804769 }, "parent": "r:15", "results": [ "s:19" ] }, { "annotation": null, "id": "r:18", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:20" ], "params": { "breakpoints": [ 0.03278688524590164, 0.03576319592827248, 0.039009688581622154, 0.04255088964328613, 0.04641355199867183, 0.050626857092594686, 0.055222635387771, 0.060235606835976424, 0.06570364317856912, 0.07166805405799236, 0.07817389910175712, 0.0852703283366209, 0.09301095350470628, 0.10145425308675969, 0.11066401409239905, 0.120709813954965, 0.13166754617157034, 0.14361999365942635, 0.15665745416000912, 0.17087842241583748, 0.186390334273537, 0.20331037833470145, 0.22176638128637152, 0.24189777359957873, 0.2638566428915663, 0.2878088829095729, 0.3139354468164648, 0.34243371424647334, 0.37351898245880216, 0.40742609285439496, 0.444411205143789, 0.4847537325694491, 0.5287584528026995, 0.5767578104625514, 0.6291144286513767, 0.6862238484814527, 0.7485175172888083, 0.8164660481095839, 0.8905827750434244, 0.9714276313637894, 1.059611379673326, 1.1558002260620701, 1.2607208531272827, 1.3751659098781202, 1.5 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 89.24639540265044, "name": "reciprocal" }, "parent": "r:17", "results": [ "s:19" ] }, { "annotation": null, "id": "r:19", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:20" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 45 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 45, 1 ] } }, "name": "reciprocal_0_45", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 45, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:18", "results": [ "s:19" ] }, { "annotation": null, "id": "r:20", "op": "attention_ops.attend_mean_where", "operands": [ "s:22", "s:29" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:15", "results": [ "s:20" ] }, { "annotation": null, "id": "r:21", "op": "attention_ops.get_prev_value", "operands": [ "s:27", "s:27" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:22" ] }, { "annotation": null, "id": "r:22", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:27" ] }, { "annotation": null, "id": "r:23", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:22", "results": [ "s:27" ] }, { "annotation": null, "id": "r:24", "op": "linear.bool_to_01", "operands": [ "s:27" ], "params": {}, "parent": null, "results": [ "s:29" ] }, { "annotation": null, "id": "r:25", "op": "linear.multiply_const", "operands": [ "s:30" ], "params": { "scalar": 0.5 }, "parent": "r:24", "results": [ "s:29" ] }, { "annotation": null, "id": "r:26", "op": "linear.add_const", "operands": [ "s:27" ], "params": { "scalar": 1.0 }, "parent": "r:24", "results": [ "s:30" ] }, { "annotation": null, "id": "r:27", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:32" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:31" ] }, { "annotation": null, "id": "r:28", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:32" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:27", "results": [ "s:31" ] }, { "annotation": null, "id": "r:29", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:32" ] }, { "annotation": null, "id": "r:30", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:62", "s:71", "s:80" ], "params": {}, "parent": null, "results": [ "s:33" ] }, { "annotation": null, "id": "r:31", "op": "linear.sum_nodes", "operands": [ "s:35", "s:63", "s:72" ], "params": { "max_fanout": null }, "parent": "r:30", "results": [ "s:33" ] }, { "annotation": null, "id": "r:32", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:62" ], "params": {}, "parent": "r:30", "results": [ "s:35" ] }, { "annotation": null, "id": "r:33", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:36" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:32", "results": [ "s:35" ] }, { "annotation": null, "id": "r:34", "op": "attention_ops.get_prev_value", "operands": [ "s:49", "s:42" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:37" ] }, { "annotation": null, "id": "r:35", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:45", "s:54" ], "params": {}, "parent": null, "results": [ "s:42" ] }, { "annotation": null, "id": "r:36", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:43" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:35", "results": [ "s:42" ] }, { "annotation": null, "id": "r:37", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:43" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:36", "results": [ "s:42" ] }, { "annotation": null, "id": "r:38", "op": "linear.sum_nodes", "operands": [ "s:45", "s:54" ], "params": { "max_fanout": null }, "parent": "r:35", "results": [ "s:43" ] }, { "annotation": null, "id": "r:39", "op": "swiglu.logic_ops.bool_any_true", "operands": [ "s:49", "s:51", "s:53" ], "params": {}, "parent": null, "results": [ "s:45" ] }, { "annotation": null, "id": "r:40", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:46" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": "r:39", "results": [ "s:45" ] }, { "annotation": null, "id": "r:41", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:46" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:40", "results": [ "s:45" ] }, { "annotation": null, "id": "r:42", "op": "linear.sum_nodes", "operands": [ "s:48", "s:50", "s:52" ], "params": { "max_fanout": null }, "parent": "r:39", "results": [ "s:46" ] }, { "annotation": null, "id": "r:43", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:49" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": "r:39", "results": [ "s:48" ] }, { "annotation": null, "id": "r:44", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:49" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:43", "results": [ "s:48" ] }, { "annotation": null, "id": "r:45", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:49" ] }, { "annotation": null, "id": "r:46", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:45", "results": [ "s:49" ] }, { "annotation": null, "id": "r:47", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:51" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": "r:39", "results": [ "s:50" ] }, { "annotation": null, "id": "r:48", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:51" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:47", "results": [ "s:50" ] }, { "annotation": null, "id": "r:49", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:51" ] }, { "annotation": null, "id": "r:50", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:49", "results": [ "s:51" ] }, { "annotation": null, "id": "r:51", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:53" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": "r:39", "results": [ "s:52" ] }, { "annotation": null, "id": "r:52", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:53" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:51", "results": [ "s:52" ] }, { "annotation": null, "id": "r:53", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:53" ] }, { "annotation": null, "id": "r:54", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:53", "results": [ "s:53" ] }, { "annotation": null, "id": "r:55", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:55" ], "params": {}, "parent": null, "results": [ "s:54" ] }, { "annotation": null, "id": "r:56", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:55" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:55", "results": [ "s:54" ] }, { "annotation": null, "id": "r:57", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:55" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:56", "results": [ "s:54" ] }, { "annotation": null, "id": "r:58", "op": "attention_ops.get_prev_value", "operands": [ "s:7", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:55" ] }, { "annotation": null, "id": "r:59", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:62" ] }, { "annotation": null, "id": "r:60", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:71" ], "params": {}, "parent": "r:30", "results": [ "s:63" ] }, { "annotation": null, "id": "r:61", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:64" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:60", "results": [ "s:63" ] }, { "annotation": null, "id": "r:62", "op": "attention_ops.get_prev_value", "operands": [ "s:51", "s:42" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:65" ] }, { "annotation": null, "id": "r:63", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:71" ] }, { "annotation": null, "id": "r:64", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:80" ], "params": {}, "parent": "r:30", "results": [ "s:72" ] }, { "annotation": null, "id": "r:65", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:73" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:64", "results": [ "s:72" ] }, { "annotation": null, "id": "r:66", "op": "attention_ops.get_prev_value", "operands": [ "s:53", "s:42" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:74" ] }, { "annotation": null, "id": "r:67", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:80" ] }, { "annotation": null, "id": "r:68", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:89" ], "params": {}, "parent": null, "results": [ "s:81" ] }, { "annotation": null, "id": "r:69", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:82" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:68", "results": [ "s:81" ] }, { "annotation": null, "id": "r:70", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:86", "s:87" ], "params": {}, "parent": null, "results": [ "s:83" ] }, { "annotation": null, "id": "r:71", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:84" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:70", "results": [ "s:83" ] }, { "annotation": null, "id": "r:72", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:84" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:71", "results": [ "s:83" ] }, { "annotation": null, "id": "r:73", "op": "linear.sum_nodes", "operands": [ "s:86", "s:87" ], "params": { "max_fanout": null }, "parent": "r:70", "results": [ "s:84" ] }, { "annotation": null, "id": "r:74", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:86" ] }, { "annotation": null, "id": "r:75", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:74", "results": [ "s:86" ] }, { "annotation": null, "id": "r:76", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:88" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -1.5, "true_level": 1.0 }, "parent": null, "results": [ "s:87" ] }, { "annotation": null, "id": "r:77", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:88" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:76", "results": [ "s:87" ] }, { "annotation": null, "id": "r:78", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:88" ] }, { "annotation": null, "id": "r:79", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:93", "s:226", "s:264" ], "params": {}, "parent": null, "results": [ "s:89" ] }, { "annotation": null, "id": "r:80", "op": "linear.sum_nodes", "operands": [ "s:91", "s:224", "s:262" ], "params": { "max_fanout": null }, "parent": "r:79", "results": [ "s:89" ] }, { "annotation": null, "id": "r:81", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:93" ], "params": {}, "parent": "r:79", "results": [ "s:91" ] }, { "annotation": null, "id": "r:82", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:92" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:81", "results": [ "s:91" ] }, { "annotation": null, "id": "r:83", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:94" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:93" ] }, { "annotation": null, "id": "r:84", "op": "linear.bool_to_01", "operands": [ "s:96" ], "params": {}, "parent": null, "results": [ "s:94" ] }, { "annotation": null, "id": "r:85", "op": "linear.multiply_const", "operands": [ "s:95" ], "params": { "scalar": 0.5 }, "parent": "r:84", "results": [ "s:94" ] }, { "annotation": null, "id": "r:86", "op": "linear.add_const", "operands": [ "s:96" ], "params": { "scalar": 1.0 }, "parent": "r:84", "results": [ "s:95" ] }, { "annotation": null, "id": "r:87", "op": "swiglu.map_select.in_range", "operands": [ "s:98", "s:223" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:96" ] }, { "annotation": null, "id": "r:88", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:97" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:87", "results": [ "s:96" ] }, { "annotation": null, "id": "r:89", "op": "linear.add", "operands": [ "s:99", "s:222" ], "params": {}, "parent": null, "results": [ "s:98" ] }, { "annotation": null, "id": "r:90", "op": "linear.add", "operands": [ "s:100", "s:165" ], "params": {}, "parent": null, "results": [ "s:99" ] }, { "annotation": null, "id": "r:91", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:101" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:100" ] }, { "annotation": null, "id": "r:92", "op": "attention_ops.get_prev_value", "operands": [ "s:107", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:101" ] }, { "annotation": null, "id": "r:93", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:113" ], "params": {}, "parent": null, "results": [ "s:107" ] }, { "annotation": null, "id": "r:94", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:108", "s:137", "s:162" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:93", "results": [ "s:107" ] }, { "annotation": null, "id": "r:95", "op": "linear.concat", "operands": [ "s:109", "s:135" ], "params": {}, "parent": "r:93", "results": [ "s:108" ] }, { "annotation": null, "id": "r:96", "op": "linear.bool_to_01", "operands": [ "s:111" ], "params": {}, "parent": "r:93", "results": [ "s:109" ] }, { "annotation": null, "id": "r:97", "op": "linear.multiply_const", "operands": [ "s:110" ], "params": { "scalar": 0.5 }, "parent": "r:96", "results": [ "s:109" ] }, { "annotation": null, "id": "r:98", "op": "linear.add_const", "operands": [ "s:111" ], "params": { "scalar": 1.0 }, "parent": "r:96", "results": [ "s:110" ] }, { "annotation": null, "id": "r:99", "op": "swiglu.map_select.in_range", "operands": [ "s:113", "s:134" ], "params": { "n_slots": 3 }, "parent": "r:93", "results": [ "s:111" ] }, { "annotation": null, "id": "r:100", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:112" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:99", "results": [ "s:111" ] }, { "annotation": null, "id": "r:101", "op": "linear.add_const", "operands": [ "s:114" ], "params": { "scalar": -1.0 }, "parent": null, "results": [ "s:113" ] }, { "annotation": null, "id": "r:102", "op": "swiglu.sequence_ops.IndexedRegion.length_at", "operands": [ "s:42" ], "params": {}, "parent": null, "results": [ "s:114" ] }, { "annotation": null, "id": "r:103", "op": "linear.add_const", "operands": [ "s:115" ], "params": { "scalar": -1.0 }, "parent": "r:102", "results": [ "s:114" ] }, { "annotation": null, "id": "r:104", "op": "attention_ops.get_prev_value", "operands": [ "s:121", "s:42" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:102", "results": [ "s:115" ] }, { "annotation": null, "id": "r:105", "op": "swiglu.sequence_ops.IndexedRegion.__init__", "operands": [ "s:8", "s:130", "s:141" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "max_len": 3, "max_read_distance": 10, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [] }, { "annotation": null, "id": "r:106", "op": "swiglu.marker_count.count_since_marker", "operands": [ "s:125", "s:132" ], "params": { "max_gap": 4, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:105", "results": [ "s:121" ] }, { "annotation": null, "id": "r:107", "op": "linear.add_const", "operands": [ "s:122" ], "params": { "scalar": -1.0 }, "parent": "r:106", "results": [ "s:121" ] }, { "annotation": null, "id": "r:108", "op": "swiglu.arithmetic_ops.reciprocal", "operands": [ "s:123" ], "params": { "max_value": 1.5, "min_value": 0.18181818181818182, "step": 0.04252199413489736 }, "parent": "r:106", "results": [ "s:122" ] }, { "annotation": null, "id": "r:109", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:123" ], "params": { "breakpoints": [ 0.18181818181818182, 0.19462576879929028, 0.20833554434393137, 0.2230110601738595, 0.23872034470395204, 0.2555362183882104, 0.2735366312792783, 0.2928050243662364, 0.3134307163656553, 0.33550931775887877, 0.35914317399480933, 0.3844418399126637, 0.4115225875838872, 0.44051094992733025, 0.4715413026176177, 0.5047574869841495, 0.5403134767881832, 0.5783740919688469, 0.6191157626666555, 0.6627273470661621, 0.7094110068488624, 0.7593831443145113, 0.8128754055148878, 0.8701357540500343, 0.9314296205045596, 0.997041132852221, 1.0672744335323334, 1.142455089303316, 1.2229316004087583, 1.3090770160517557, 1.4012906636660458, 1.5 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 20.739660046174524, "name": "reciprocal" }, "parent": "r:108", "results": [ "s:122" ] }, { "annotation": null, "id": "r:110", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:123" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 32 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "name": "reciprocal_0_32", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:109", "results": [ "s:122" ] }, { "annotation": null, "id": "r:111", "op": "attention_ops.attend_mean_where", "operands": [ "s:125", "s:132" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:106", "results": [ "s:123" ] }, { "annotation": null, "id": "r:112", "op": "attention_ops.get_prev_value", "operands": [ "s:130", "s:130" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:105", "results": [ "s:125" ] }, { "annotation": null, "id": "r:113", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:130" ] }, { "annotation": null, "id": "r:114", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:113", "results": [ "s:130" ] }, { "annotation": null, "id": "r:115", "op": "linear.bool_to_01", "operands": [ "s:130" ], "params": {}, "parent": "r:105", "results": [ "s:132" ] }, { "annotation": null, "id": "r:116", "op": "linear.multiply_const", "operands": [ "s:133" ], "params": { "scalar": 0.5 }, "parent": "r:115", "results": [ "s:132" ] }, { "annotation": null, "id": "r:117", "op": "linear.add_const", "operands": [ "s:130" ], "params": { "scalar": 1.0 }, "parent": "r:115", "results": [ "s:133" ] }, { "annotation": null, "id": "r:118", "op": "linear.add_const", "operands": [ "s:113" ], "params": { "scalar": 1.0 }, "parent": "r:93", "results": [ "s:134" ] }, { "annotation": null, "id": "r:119", "op": "linear.add_const", "operands": [ "s:136" ], "params": { "scalar": 1.0 }, "parent": "r:93", "results": [ "s:135" ] }, { "annotation": null, "id": "r:120", "op": "linear.add", "operands": [ "s:138", "s:159" ], "params": {}, "parent": "r:105", "results": [ "s:137" ] }, { "annotation": null, "id": "r:121", "op": "linear.concat", "operands": [ "s:139", "s:158" ], "params": {}, "parent": "r:105", "results": [ "s:138" ] }, { "annotation": null, "id": "r:122", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:141", "s:152" ], "params": {}, "parent": "r:105", "results": [ "s:139" ] }, { "annotation": null, "id": "r:123", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:140" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 4 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 3 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 4 ] } } }, "parent": "r:122", "results": [ "s:139" ] }, { "annotation": null, "id": "r:124", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:144", "s:145" ], "params": {}, "parent": null, "results": [ "s:141" ] }, { "annotation": null, "id": "r:125", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:142" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:124", "results": [ "s:141" ] }, { "annotation": null, "id": "r:126", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:142" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:125", "results": [ "s:141" ] }, { "annotation": null, "id": "r:127", "op": "linear.sum_nodes", "operands": [ "s:144", "s:145" ], "params": { "max_fanout": null }, "parent": "r:124", "results": [ "s:142" ] }, { "annotation": null, "id": "r:128", "op": "swiglu.sequence_ops.check_is_digit", "operands": [ "s:8" ], "params": {}, "parent": null, "results": [ "s:144" ] }, { "annotation": null, "id": "r:129", "op": "swiglu.map_select.map_to_table", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": "r:128", "results": [ "s:144" ] }, { "annotation": null, "id": "r:130", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 10 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 10, 33 ] } }, "name": "map_to_table", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 10, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:129", "results": [ "s:144" ] }, { "annotation": null, "id": "r:131", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:146" ], "params": {}, "parent": null, "results": [ "s:145" ] }, { "annotation": null, "id": "r:132", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:146" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:131", "results": [ "s:145" ] }, { "annotation": null, "id": "r:133", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:146" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:132", "results": [ "s:145" ] }, { "annotation": null, "id": "r:134", "op": "attention_ops.get_prev_value", "operands": [ "s:42", "s:42" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:146" ] }, { "annotation": null, "id": "r:135", "op": "linear.bool_to_01", "operands": [ "s:154" ], "params": {}, "parent": "r:105", "results": [ "s:152" ] }, { "annotation": null, "id": "r:136", "op": "linear.multiply_const", "operands": [ "s:153" ], "params": { "scalar": 0.5 }, "parent": "r:135", "results": [ "s:152" ] }, { "annotation": null, "id": "r:137", "op": "linear.add_const", "operands": [ "s:154" ], "params": { "scalar": 1.0 }, "parent": "r:135", "results": [ "s:153" ] }, { "annotation": null, "id": "r:138", "op": "swiglu.map_select.in_range", "operands": [ "s:156", "s:157" ], "params": { "n_slots": 3 }, "parent": "r:105", "results": [ "s:154" ] }, { "annotation": null, "id": "r:139", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:155" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:138", "results": [ "s:154" ] }, { "annotation": null, "id": "r:140", "op": "linear.add_const", "operands": [ "s:121" ], "params": { "scalar": -1.0 }, "parent": "r:105", "results": [ "s:156" ] }, { "annotation": null, "id": "r:141", "op": "linear.add_const", "operands": [ "s:156" ], "params": { "scalar": 1.0 }, "parent": "r:105", "results": [ "s:157" ] }, { "annotation": null, "id": "r:142", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": "r:105", "results": [ "s:158" ] }, { "annotation": null, "id": "r:143", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:130", "s:161" ], "params": {}, "parent": "r:105", "results": [ "s:159" ] }, { "annotation": null, "id": "r:144", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:160" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 5 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 4 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 5 ] } } }, "parent": "r:143", "results": [ "s:159" ] }, { "annotation": null, "id": "r:145", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 4 ] } } }, "parent": "r:105", "results": [ "s:161" ] }, { "annotation": null, "id": "r:146", "op": "swiglu.map_select.select", "operands": [ "s:141", "s:8", "s:164" ], "params": {}, "parent": "r:105", "results": [ "s:162" ] }, { "annotation": null, "id": "r:147", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:163" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:146", "results": [ "s:162" ] }, { "annotation": null, "id": "r:148", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": "r:105", "results": [ "s:164" ] }, { "annotation": null, "id": "r:149", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:166" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:165" ] }, { "annotation": null, "id": "r:150", "op": "attention_ops.get_prev_value", "operands": [ "s:172", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:166" ] }, { "annotation": null, "id": "r:151", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:178" ], "params": {}, "parent": null, "results": [ "s:172" ] }, { "annotation": null, "id": "r:152", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:173", "s:201", "s:219" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:151", "results": [ "s:172" ] }, { "annotation": null, "id": "r:153", "op": "linear.concat", "operands": [ "s:174", "s:199" ], "params": {}, "parent": "r:151", "results": [ "s:173" ] }, { "annotation": null, "id": "r:154", "op": "linear.bool_to_01", "operands": [ "s:176" ], "params": {}, "parent": "r:151", "results": [ "s:174" ] }, { "annotation": null, "id": "r:155", "op": "linear.multiply_const", "operands": [ "s:175" ], "params": { "scalar": 0.5 }, "parent": "r:154", "results": [ "s:174" ] }, { "annotation": null, "id": "r:156", "op": "linear.add_const", "operands": [ "s:176" ], "params": { "scalar": 1.0 }, "parent": "r:154", "results": [ "s:175" ] }, { "annotation": null, "id": "r:157", "op": "swiglu.map_select.in_range", "operands": [ "s:178", "s:198" ], "params": { "n_slots": 3 }, "parent": "r:151", "results": [ "s:176" ] }, { "annotation": null, "id": "r:158", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:177" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:157", "results": [ "s:176" ] }, { "annotation": null, "id": "r:159", "op": "linear.add_const", "operands": [ "s:179" ], "params": { "scalar": -1.0 }, "parent": null, "results": [ "s:178" ] }, { "annotation": null, "id": "r:160", "op": "swiglu.sequence_ops.IndexedRegion.length_at", "operands": [ "s:7" ], "params": {}, "parent": null, "results": [ "s:179" ] }, { "annotation": null, "id": "r:161", "op": "linear.add_const", "operands": [ "s:180" ], "params": { "scalar": -1.0 }, "parent": "r:160", "results": [ "s:179" ] }, { "annotation": null, "id": "r:162", "op": "attention_ops.get_prev_value", "operands": [ "s:186", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:160", "results": [ "s:180" ] }, { "annotation": null, "id": "r:163", "op": "swiglu.sequence_ops.IndexedRegion.__init__", "operands": [ "s:8", "s:42", "s:205" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "max_len": 3, "max_read_distance": 10, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [] }, { "annotation": null, "id": "r:164", "op": "swiglu.marker_count.count_since_marker", "operands": [ "s:190", "s:196" ], "params": { "max_gap": 4, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:163", "results": [ "s:186" ] }, { "annotation": null, "id": "r:165", "op": "linear.add_const", "operands": [ "s:187" ], "params": { "scalar": -1.0 }, "parent": "r:164", "results": [ "s:186" ] }, { "annotation": null, "id": "r:166", "op": "swiglu.arithmetic_ops.reciprocal", "operands": [ "s:188" ], "params": { "max_value": 1.5, "min_value": 0.18181818181818182, "step": 0.04252199413489736 }, "parent": "r:164", "results": [ "s:187" ] }, { "annotation": null, "id": "r:167", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:188" ], "params": { "breakpoints": [ 0.18181818181818182, 0.19462576879929028, 0.20833554434393137, 0.2230110601738595, 0.23872034470395204, 0.2555362183882104, 0.2735366312792783, 0.2928050243662364, 0.3134307163656553, 0.33550931775887877, 0.35914317399480933, 0.3844418399126637, 0.4115225875838872, 0.44051094992733025, 0.4715413026176177, 0.5047574869841495, 0.5403134767881832, 0.5783740919688469, 0.6191157626666555, 0.6627273470661621, 0.7094110068488624, 0.7593831443145113, 0.8128754055148878, 0.8701357540500343, 0.9314296205045596, 0.997041132852221, 1.0672744335323334, 1.142455089303316, 1.2229316004087583, 1.3090770160517557, 1.4012906636660458, 1.5 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 20.739660046174524, "name": "reciprocal" }, "parent": "r:166", "results": [ "s:187" ] }, { "annotation": null, "id": "r:168", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:188" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 32 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "name": "reciprocal_0_32", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:167", "results": [ "s:187" ] }, { "annotation": null, "id": "r:169", "op": "attention_ops.attend_mean_where", "operands": [ "s:190", "s:196" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:164", "results": [ "s:188" ] }, { "annotation": null, "id": "r:170", "op": "attention_ops.get_prev_value", "operands": [ "s:42", "s:42" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:163", "results": [ "s:190" ] }, { "annotation": null, "id": "r:171", "op": "linear.bool_to_01", "operands": [ "s:42" ], "params": {}, "parent": "r:163", "results": [ "s:196" ] }, { "annotation": null, "id": "r:172", "op": "linear.multiply_const", "operands": [ "s:197" ], "params": { "scalar": 0.5 }, "parent": "r:171", "results": [ "s:196" ] }, { "annotation": null, "id": "r:173", "op": "linear.add_const", "operands": [ "s:42" ], "params": { "scalar": 1.0 }, "parent": "r:171", "results": [ "s:197" ] }, { "annotation": null, "id": "r:174", "op": "linear.add_const", "operands": [ "s:178" ], "params": { "scalar": 1.0 }, "parent": "r:151", "results": [ "s:198" ] }, { "annotation": null, "id": "r:175", "op": "linear.add_const", "operands": [ "s:200" ], "params": { "scalar": 1.0 }, "parent": "r:151", "results": [ "s:199" ] }, { "annotation": null, "id": "r:176", "op": "linear.add", "operands": [ "s:202", "s:216" ], "params": {}, "parent": "r:163", "results": [ "s:201" ] }, { "annotation": null, "id": "r:177", "op": "linear.concat", "operands": [ "s:203", "s:215" ], "params": {}, "parent": "r:163", "results": [ "s:202" ] }, { "annotation": null, "id": "r:178", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:205", "s:209" ], "params": {}, "parent": "r:163", "results": [ "s:203" ] }, { "annotation": null, "id": "r:179", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:204" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 4 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 3 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 4 ] } } }, "parent": "r:178", "results": [ "s:203" ] }, { "annotation": null, "id": "r:180", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:144", "s:146", "s:208" ], "params": {}, "parent": null, "results": [ "s:205" ] }, { "annotation": null, "id": "r:181", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:206" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 2.0, "true_level": 1.0 }, "parent": "r:180", "results": [ "s:205" ] }, { "annotation": null, "id": "r:182", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:206" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:181", "results": [ "s:205" ] }, { "annotation": null, "id": "r:183", "op": "linear.sum_nodes", "operands": [ "s:144", "s:146", "s:208" ], "params": { "max_fanout": null }, "parent": "r:180", "results": [ "s:206" ] }, { "annotation": null, "id": "r:184", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:55" ], "params": {}, "parent": null, "results": [ "s:208" ] }, { "annotation": null, "id": "r:185", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:55" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:184", "results": [ "s:208" ] }, { "annotation": null, "id": "r:186", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:55" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:185", "results": [ "s:208" ] }, { "annotation": null, "id": "r:187", "op": "linear.bool_to_01", "operands": [ "s:211" ], "params": {}, "parent": "r:163", "results": [ "s:209" ] }, { "annotation": null, "id": "r:188", "op": "linear.multiply_const", "operands": [ "s:210" ], "params": { "scalar": 0.5 }, "parent": "r:187", "results": [ "s:209" ] }, { "annotation": null, "id": "r:189", "op": "linear.add_const", "operands": [ "s:211" ], "params": { "scalar": 1.0 }, "parent": "r:187", "results": [ "s:210" ] }, { "annotation": null, "id": "r:190", "op": "swiglu.map_select.in_range", "operands": [ "s:213", "s:214" ], "params": { "n_slots": 3 }, "parent": "r:163", "results": [ "s:211" ] }, { "annotation": null, "id": "r:191", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:212" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:190", "results": [ "s:211" ] }, { "annotation": null, "id": "r:192", "op": "linear.add_const", "operands": [ "s:186" ], "params": { "scalar": -1.0 }, "parent": "r:163", "results": [ "s:213" ] }, { "annotation": null, "id": "r:193", "op": "linear.add_const", "operands": [ "s:213" ], "params": { "scalar": 1.0 }, "parent": "r:163", "results": [ "s:214" ] }, { "annotation": null, "id": "r:194", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": "r:163", "results": [ "s:215" ] }, { "annotation": null, "id": "r:195", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:42", "s:218" ], "params": {}, "parent": "r:163", "results": [ "s:216" ] }, { "annotation": null, "id": "r:196", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:217" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 5 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 4 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 5 ] } } }, "parent": "r:195", "results": [ "s:216" ] }, { "annotation": null, "id": "r:197", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 4 ] } } }, "parent": "r:163", "results": [ "s:218" ] }, { "annotation": null, "id": "r:198", "op": "swiglu.map_select.select", "operands": [ "s:205", "s:8", "s:221" ], "params": {}, "parent": "r:163", "results": [ "s:219" ] }, { "annotation": null, "id": "r:199", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:220" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:198", "results": [ "s:219" ] }, { "annotation": null, "id": "r:200", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": "r:163", "results": [ "s:221" ] }, { "annotation": null, "id": "r:201", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:222" ] }, { "annotation": null, "id": "r:202", "op": "linear.add_const", "operands": [ "s:98" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:223" ] }, { "annotation": null, "id": "r:203", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:226" ], "params": {}, "parent": "r:79", "results": [ "s:224" ] }, { "annotation": null, "id": "r:204", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:225" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:203", "results": [ "s:224" ] }, { "annotation": null, "id": "r:205", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:227" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:226" ] }, { "annotation": null, "id": "r:206", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:228" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:227" ] }, { "annotation": null, "id": "r:207", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:228" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 300 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 300, 69 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 300, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:206", "results": [ "s:227" ] }, { "annotation": null, "id": "r:208", "op": "linear.concat", "operands": [ "s:229", "s:230", "s:246" ], "params": {}, "parent": null, "results": [ "s:228" ] }, { "annotation": null, "id": "r:209", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:229" ] }, { "annotation": null, "id": "r:210", "op": "attention_ops.get_prev_value", "operands": [ "s:236", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:230" ] }, { "annotation": null, "id": "r:211", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:242" ], "params": {}, "parent": null, "results": [ "s:236" ] }, { "annotation": null, "id": "r:212", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:237", "s:137", "s:162" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:211", "results": [ "s:236" ] }, { "annotation": null, "id": "r:213", "op": "linear.concat", "operands": [ "s:238", "s:244" ], "params": {}, "parent": "r:211", "results": [ "s:237" ] }, { "annotation": null, "id": "r:214", "op": "linear.bool_to_01", "operands": [ "s:240" ], "params": {}, "parent": "r:211", "results": [ "s:238" ] }, { "annotation": null, "id": "r:215", "op": "linear.multiply_const", "operands": [ "s:239" ], "params": { "scalar": 0.5 }, "parent": "r:214", "results": [ "s:238" ] }, { "annotation": null, "id": "r:216", "op": "linear.add_const", "operands": [ "s:240" ], "params": { "scalar": 1.0 }, "parent": "r:214", "results": [ "s:239" ] }, { "annotation": null, "id": "r:217", "op": "swiglu.map_select.in_range", "operands": [ "s:242", "s:243" ], "params": { "n_slots": 3 }, "parent": "r:211", "results": [ "s:240" ] }, { "annotation": null, "id": "r:218", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:241" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:217", "results": [ "s:240" ] }, { "annotation": null, "id": "r:219", "op": "linear.add_const", "operands": [ "s:114" ], "params": { "scalar": -3.0 }, "parent": null, "results": [ "s:242" ] }, { "annotation": null, "id": "r:220", "op": "linear.add_const", "operands": [ "s:242" ], "params": { "scalar": 1.0 }, "parent": "r:211", "results": [ "s:243" ] }, { "annotation": null, "id": "r:221", "op": "linear.add_const", "operands": [ "s:245" ], "params": { "scalar": 1.0 }, "parent": "r:211", "results": [ "s:244" ] }, { "annotation": null, "id": "r:222", "op": "attention_ops.get_prev_value", "operands": [ "s:252", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:246" ] }, { "annotation": null, "id": "r:223", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:258" ], "params": {}, "parent": null, "results": [ "s:252" ] }, { "annotation": null, "id": "r:224", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:253", "s:201", "s:219" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:223", "results": [ "s:252" ] }, { "annotation": null, "id": "r:225", "op": "linear.concat", "operands": [ "s:254", "s:260" ], "params": {}, "parent": "r:223", "results": [ "s:253" ] }, { "annotation": null, "id": "r:226", "op": "linear.bool_to_01", "operands": [ "s:256" ], "params": {}, "parent": "r:223", "results": [ "s:254" ] }, { "annotation": null, "id": "r:227", "op": "linear.multiply_const", "operands": [ "s:255" ], "params": { "scalar": 0.5 }, "parent": "r:226", "results": [ "s:254" ] }, { "annotation": null, "id": "r:228", "op": "linear.add_const", "operands": [ "s:256" ], "params": { "scalar": 1.0 }, "parent": "r:226", "results": [ "s:255" ] }, { "annotation": null, "id": "r:229", "op": "swiglu.map_select.in_range", "operands": [ "s:258", "s:259" ], "params": { "n_slots": 3 }, "parent": "r:223", "results": [ "s:256" ] }, { "annotation": null, "id": "r:230", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:257" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:229", "results": [ "s:256" ] }, { "annotation": null, "id": "r:231", "op": "linear.add_const", "operands": [ "s:179" ], "params": { "scalar": -3.0 }, "parent": null, "results": [ "s:258" ] }, { "annotation": null, "id": "r:232", "op": "linear.add_const", "operands": [ "s:258" ], "params": { "scalar": 1.0 }, "parent": "r:223", "results": [ "s:259" ] }, { "annotation": null, "id": "r:233", "op": "linear.add_const", "operands": [ "s:261" ], "params": { "scalar": 1.0 }, "parent": "r:223", "results": [ "s:260" ] }, { "annotation": null, "id": "r:234", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:264" ], "params": {}, "parent": "r:79", "results": [ "s:262" ] }, { "annotation": null, "id": "r:235", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:263" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:234", "results": [ "s:262" ] }, { "annotation": null, "id": "r:236", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:265" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:264" ] }, { "annotation": null, "id": "r:237", "op": "linear.bool_to_01", "operands": [ "s:267" ], "params": {}, "parent": null, "results": [ "s:265" ] }, { "annotation": null, "id": "r:238", "op": "linear.multiply_const", "operands": [ "s:266" ], "params": { "scalar": 0.5 }, "parent": "r:237", "results": [ "s:265" ] }, { "annotation": null, "id": "r:239", "op": "linear.add_const", "operands": [ "s:267" ], "params": { "scalar": 1.0 }, "parent": "r:237", "results": [ "s:266" ] }, { "annotation": null, "id": "r:240", "op": "swiglu.map_select.in_range", "operands": [ "s:269", "s:276" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:267" ] }, { "annotation": null, "id": "r:241", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:268" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:240", "results": [ "s:267" ] }, { "annotation": null, "id": "r:242", "op": "linear.add", "operands": [ "s:270", "s:275" ], "params": {}, "parent": null, "results": [ "s:269" ] }, { "annotation": null, "id": "r:243", "op": "linear.sum_nodes", "operands": [ "s:272" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:270" ] }, { "annotation": null, "id": "r:244", "op": "linear.slice_columns", "operands": [ "s:273" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:272" ] }, { "annotation": null, "id": "r:245", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:274" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:273" ] }, { "annotation": null, "id": "r:246", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:274" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 66 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:245", "results": [ "s:273" ] }, { "annotation": null, "id": "r:247", "op": "linear.concat", "operands": [ "s:101", "s:166" ], "params": {}, "parent": null, "results": [ "s:274" ] }, { "annotation": null, "id": "r:248", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:275" ] }, { "annotation": null, "id": "r:249", "op": "linear.add_const", "operands": [ "s:269" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:276" ] }, { "annotation": null, "id": "r:250", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:279", "s:285" ], "params": {}, "parent": null, "results": [ "s:277" ] }, { "annotation": null, "id": "r:251", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:278" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:250", "results": [ "s:277" ] }, { "annotation": null, "id": "r:252", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:282", "s:283" ], "params": {}, "parent": null, "results": [ "s:279" ] }, { "annotation": null, "id": "r:253", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:280" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:252", "results": [ "s:279" ] }, { "annotation": null, "id": "r:254", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:280" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:253", "results": [ "s:279" ] }, { "annotation": null, "id": "r:255", "op": "linear.sum_nodes", "operands": [ "s:282", "s:283" ], "params": { "max_fanout": null }, "parent": "r:252", "results": [ "s:280" ] }, { "annotation": null, "id": "r:256", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.5, "true_level": 1.0 }, "parent": null, "results": [ "s:282" ] }, { "annotation": null, "id": "r:257", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:256", "results": [ "s:282" ] }, { "annotation": null, "id": "r:258", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:284" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -2.5, "true_level": 1.0 }, "parent": null, "results": [ "s:283" ] }, { "annotation": null, "id": "r:259", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:284" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:258", "results": [ "s:283" ] }, { "annotation": null, "id": "r:260", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:284" ] }, { "annotation": null, "id": "r:261", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:289", "s:334", "s:339" ], "params": {}, "parent": null, "results": [ "s:285" ] }, { "annotation": null, "id": "r:262", "op": "linear.sum_nodes", "operands": [ "s:287", "s:332", "s:337" ], "params": { "max_fanout": null }, "parent": "r:261", "results": [ "s:285" ] }, { "annotation": null, "id": "r:263", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:289" ], "params": {}, "parent": "r:261", "results": [ "s:287" ] }, { "annotation": null, "id": "r:264", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:288" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:263", "results": [ "s:287" ] }, { "annotation": null, "id": "r:265", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:290" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:289" ] }, { "annotation": null, "id": "r:266", "op": "linear.bool_to_01", "operands": [ "s:292" ], "params": {}, "parent": null, "results": [ "s:290" ] }, { "annotation": null, "id": "r:267", "op": "linear.multiply_const", "operands": [ "s:291" ], "params": { "scalar": 0.5 }, "parent": "r:266", "results": [ "s:290" ] }, { "annotation": null, "id": "r:268", "op": "linear.add_const", "operands": [ "s:292" ], "params": { "scalar": 1.0 }, "parent": "r:266", "results": [ "s:291" ] }, { "annotation": null, "id": "r:269", "op": "swiglu.map_select.in_range", "operands": [ "s:294", "s:331" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:292" ] }, { "annotation": null, "id": "r:270", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:293" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:269", "results": [ "s:292" ] }, { "annotation": null, "id": "r:271", "op": "linear.add", "operands": [ "s:295", "s:330" ], "params": {}, "parent": null, "results": [ "s:294" ] }, { "annotation": null, "id": "r:272", "op": "linear.add", "operands": [ "s:296", "s:313" ], "params": {}, "parent": null, "results": [ "s:295" ] }, { "annotation": null, "id": "r:273", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:297" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:296" ] }, { "annotation": null, "id": "r:274", "op": "attention_ops.get_prev_value", "operands": [ "s:303", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:297" ] }, { "annotation": null, "id": "r:275", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:309" ], "params": {}, "parent": null, "results": [ "s:303" ] }, { "annotation": null, "id": "r:276", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:304", "s:137", "s:162" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:275", "results": [ "s:303" ] }, { "annotation": null, "id": "r:277", "op": "linear.concat", "operands": [ "s:305", "s:311" ], "params": {}, "parent": "r:275", "results": [ "s:304" ] }, { "annotation": null, "id": "r:278", "op": "linear.bool_to_01", "operands": [ "s:307" ], "params": {}, "parent": "r:275", "results": [ "s:305" ] }, { "annotation": null, "id": "r:279", "op": "linear.multiply_const", "operands": [ "s:306" ], "params": { "scalar": 0.5 }, "parent": "r:278", "results": [ "s:305" ] }, { "annotation": null, "id": "r:280", "op": "linear.add_const", "operands": [ "s:307" ], "params": { "scalar": 1.0 }, "parent": "r:278", "results": [ "s:306" ] }, { "annotation": null, "id": "r:281", "op": "swiglu.map_select.in_range", "operands": [ "s:309", "s:310" ], "params": { "n_slots": 3 }, "parent": "r:275", "results": [ "s:307" ] }, { "annotation": null, "id": "r:282", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:308" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:281", "results": [ "s:307" ] }, { "annotation": null, "id": "r:283", "op": "linear.add_const", "operands": [ "s:114" ], "params": { "scalar": -2.0 }, "parent": null, "results": [ "s:309" ] }, { "annotation": null, "id": "r:284", "op": "linear.add_const", "operands": [ "s:309" ], "params": { "scalar": 1.0 }, "parent": "r:275", "results": [ "s:310" ] }, { "annotation": null, "id": "r:285", "op": "linear.add_const", "operands": [ "s:312" ], "params": { "scalar": 1.0 }, "parent": "r:275", "results": [ "s:311" ] }, { "annotation": null, "id": "r:286", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:314" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:313" ] }, { "annotation": null, "id": "r:287", "op": "attention_ops.get_prev_value", "operands": [ "s:320", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:314" ] }, { "annotation": null, "id": "r:288", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:326" ], "params": {}, "parent": null, "results": [ "s:320" ] }, { "annotation": null, "id": "r:289", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:321", "s:201", "s:219" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:288", "results": [ "s:320" ] }, { "annotation": null, "id": "r:290", "op": "linear.concat", "operands": [ "s:322", "s:328" ], "params": {}, "parent": "r:288", "results": [ "s:321" ] }, { "annotation": null, "id": "r:291", "op": "linear.bool_to_01", "operands": [ "s:324" ], "params": {}, "parent": "r:288", "results": [ "s:322" ] }, { "annotation": null, "id": "r:292", "op": "linear.multiply_const", "operands": [ "s:323" ], "params": { "scalar": 0.5 }, "parent": "r:291", "results": [ "s:322" ] }, { "annotation": null, "id": "r:293", "op": "linear.add_const", "operands": [ "s:324" ], "params": { "scalar": 1.0 }, "parent": "r:291", "results": [ "s:323" ] }, { "annotation": null, "id": "r:294", "op": "swiglu.map_select.in_range", "operands": [ "s:326", "s:327" ], "params": { "n_slots": 3 }, "parent": "r:288", "results": [ "s:324" ] }, { "annotation": null, "id": "r:295", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:325" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:294", "results": [ "s:324" ] }, { "annotation": null, "id": "r:296", "op": "linear.add_const", "operands": [ "s:179" ], "params": { "scalar": -2.0 }, "parent": null, "results": [ "s:326" ] }, { "annotation": null, "id": "r:297", "op": "linear.add_const", "operands": [ "s:326" ], "params": { "scalar": 1.0 }, "parent": "r:288", "results": [ "s:327" ] }, { "annotation": null, "id": "r:298", "op": "linear.add_const", "operands": [ "s:329" ], "params": { "scalar": 1.0 }, "parent": "r:288", "results": [ "s:328" ] }, { "annotation": null, "id": "r:299", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:330" ] }, { "annotation": null, "id": "r:300", "op": "linear.add_const", "operands": [ "s:294" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:331" ] }, { "annotation": null, "id": "r:301", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:334" ], "params": {}, "parent": "r:261", "results": [ "s:332" ] }, { "annotation": null, "id": "r:302", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:333" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:301", "results": [ "s:332" ] }, { "annotation": null, "id": "r:303", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:335" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:334" ] }, { "annotation": null, "id": "r:304", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:336" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:335" ] }, { "annotation": null, "id": "r:305", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:336" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 300 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 300, 69 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 300, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:304", "results": [ "s:335" ] }, { "annotation": null, "id": "r:306", "op": "linear.concat", "operands": [ "s:229", "s:297", "s:314" ], "params": {}, "parent": null, "results": [ "s:336" ] }, { "annotation": null, "id": "r:307", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:339" ], "params": {}, "parent": "r:261", "results": [ "s:337" ] }, { "annotation": null, "id": "r:308", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:338" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:307", "results": [ "s:337" ] }, { "annotation": null, "id": "r:309", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:340" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:339" ] }, { "annotation": null, "id": "r:310", "op": "linear.bool_to_01", "operands": [ "s:342" ], "params": {}, "parent": null, "results": [ "s:340" ] }, { "annotation": null, "id": "r:311", "op": "linear.multiply_const", "operands": [ "s:341" ], "params": { "scalar": 0.5 }, "parent": "r:310", "results": [ "s:340" ] }, { "annotation": null, "id": "r:312", "op": "linear.add_const", "operands": [ "s:342" ], "params": { "scalar": 1.0 }, "parent": "r:310", "results": [ "s:341" ] }, { "annotation": null, "id": "r:313", "op": "swiglu.map_select.in_range", "operands": [ "s:344", "s:355" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:342" ] }, { "annotation": null, "id": "r:314", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:343" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:313", "results": [ "s:342" ] }, { "annotation": null, "id": "r:315", "op": "linear.add", "operands": [ "s:345", "s:354" ], "params": {}, "parent": null, "results": [ "s:344" ] }, { "annotation": null, "id": "r:316", "op": "linear.sum_nodes", "operands": [ "s:347", "s:350", "s:353" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:345" ] }, { "annotation": null, "id": "r:317", "op": "linear.slice_columns", "operands": [ "s:348" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:347" ] }, { "annotation": null, "id": "r:318", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:349" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:348" ] }, { "annotation": null, "id": "r:319", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:349" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 66 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:318", "results": [ "s:348" ] }, { "annotation": null, "id": "r:320", "op": "linear.concat", "operands": [ "s:297", "s:166" ], "params": {}, "parent": null, "results": [ "s:349" ] }, { "annotation": null, "id": "r:321", "op": "linear.slice_columns", "operands": [ "s:351" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:350" ] }, { "annotation": null, "id": "r:322", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:352" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:351" ] }, { "annotation": null, "id": "r:323", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:352" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 66 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:322", "results": [ "s:351" ] }, { "annotation": null, "id": "r:324", "op": "linear.concat", "operands": [ "s:101", "s:314" ], "params": {}, "parent": null, "results": [ "s:352" ] }, { "annotation": null, "id": "r:325", "op": "linear.slice_columns", "operands": [ "s:273" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:353" ] }, { "annotation": null, "id": "r:326", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:354" ] }, { "annotation": null, "id": "r:327", "op": "linear.add_const", "operands": [ "s:344" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:355" ] }, { "annotation": null, "id": "r:328", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:358", "s:364" ], "params": {}, "parent": null, "results": [ "s:356" ] }, { "annotation": null, "id": "r:329", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:357" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:328", "results": [ "s:356" ] }, { "annotation": null, "id": "r:330", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:361", "s:362" ], "params": {}, "parent": null, "results": [ "s:358" ] }, { "annotation": null, "id": "r:331", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:359" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:330", "results": [ "s:358" ] }, { "annotation": null, "id": "r:332", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:359" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:331", "results": [ "s:358" ] }, { "annotation": null, "id": "r:333", "op": "linear.sum_nodes", "operands": [ "s:361", "s:362" ], "params": { "max_fanout": null }, "parent": "r:330", "results": [ "s:359" ] }, { "annotation": null, "id": "r:334", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 2.5, "true_level": 1.0 }, "parent": null, "results": [ "s:361" ] }, { "annotation": null, "id": "r:335", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:334", "results": [ "s:361" ] }, { "annotation": null, "id": "r:336", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:363" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -3.5, "true_level": 1.0 }, "parent": null, "results": [ "s:362" ] }, { "annotation": null, "id": "r:337", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:363" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:336", "results": [ "s:362" ] }, { "annotation": null, "id": "r:338", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:363" ] }, { "annotation": null, "id": "r:339", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:368", "s:380", "s:385" ], "params": {}, "parent": null, "results": [ "s:364" ] }, { "annotation": null, "id": "r:340", "op": "linear.sum_nodes", "operands": [ "s:366", "s:378", "s:383" ], "params": { "max_fanout": null }, "parent": "r:339", "results": [ "s:364" ] }, { "annotation": null, "id": "r:341", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:368" ], "params": {}, "parent": "r:339", "results": [ "s:366" ] }, { "annotation": null, "id": "r:342", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:367" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:341", "results": [ "s:366" ] }, { "annotation": null, "id": "r:343", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:369" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:368" ] }, { "annotation": null, "id": "r:344", "op": "linear.bool_to_01", "operands": [ "s:371" ], "params": {}, "parent": null, "results": [ "s:369" ] }, { "annotation": null, "id": "r:345", "op": "linear.multiply_const", "operands": [ "s:370" ], "params": { "scalar": 0.5 }, "parent": "r:344", "results": [ "s:369" ] }, { "annotation": null, "id": "r:346", "op": "linear.add_const", "operands": [ "s:371" ], "params": { "scalar": 1.0 }, "parent": "r:344", "results": [ "s:370" ] }, { "annotation": null, "id": "r:347", "op": "swiglu.map_select.in_range", "operands": [ "s:373", "s:377" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:371" ] }, { "annotation": null, "id": "r:348", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:372" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:347", "results": [ "s:371" ] }, { "annotation": null, "id": "r:349", "op": "linear.add", "operands": [ "s:374", "s:330" ], "params": {}, "parent": null, "results": [ "s:373" ] }, { "annotation": null, "id": "r:350", "op": "linear.add", "operands": [ "s:375", "s:376" ], "params": {}, "parent": null, "results": [ "s:374" ] }, { "annotation": null, "id": "r:351", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:230" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:375" ] }, { "annotation": null, "id": "r:352", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:246" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:376" ] }, { "annotation": null, "id": "r:353", "op": "linear.add_const", "operands": [ "s:373" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:377" ] }, { "annotation": null, "id": "r:354", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:380" ], "params": {}, "parent": "r:339", "results": [ "s:378" ] }, { "annotation": null, "id": "r:355", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:379" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:354", "results": [ "s:378" ] }, { "annotation": null, "id": "r:356", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:381" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:380" ] }, { "annotation": null, "id": "r:357", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:382" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:381" ] }, { "annotation": null, "id": "r:358", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:382" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 300 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 300, 69 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 300, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:357", "results": [ "s:381" ] }, { "annotation": null, "id": "r:359", "op": "linear.concat", "operands": [ "s:229", "s:101", "s:166" ], "params": {}, "parent": null, "results": [ "s:382" ] }, { "annotation": null, "id": "r:360", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:385" ], "params": {}, "parent": "r:339", "results": [ "s:383" ] }, { "annotation": null, "id": "r:361", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:384" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:360", "results": [ "s:383" ] }, { "annotation": null, "id": "r:362", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:386" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:385" ] }, { "annotation": null, "id": "r:363", "op": "linear.bool_to_01", "operands": [ "s:388" ], "params": {}, "parent": null, "results": [ "s:386" ] }, { "annotation": null, "id": "r:364", "op": "linear.multiply_const", "operands": [ "s:387" ], "params": { "scalar": 0.5 }, "parent": "r:363", "results": [ "s:386" ] }, { "annotation": null, "id": "r:365", "op": "linear.add_const", "operands": [ "s:388" ], "params": { "scalar": 1.0 }, "parent": "r:363", "results": [ "s:387" ] }, { "annotation": null, "id": "r:366", "op": "swiglu.map_select.in_range", "operands": [ "s:390", "s:404" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:388" ] }, { "annotation": null, "id": "r:367", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:389" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:366", "results": [ "s:388" ] }, { "annotation": null, "id": "r:368", "op": "linear.add", "operands": [ "s:391", "s:354" ], "params": {}, "parent": null, "results": [ "s:390" ] }, { "annotation": null, "id": "r:369", "op": "linear.sum_nodes", "operands": [ "s:393", "s:396", "s:399", "s:400", "s:403" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:391" ] }, { "annotation": null, "id": "r:370", "op": "linear.slice_columns", "operands": [ "s:394" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:393" ] }, { "annotation": null, "id": "r:371", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:395" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:394" ] }, { "annotation": null, "id": "r:372", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:395" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 66 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:371", "results": [ "s:394" ] }, { "annotation": null, "id": "r:373", "op": "linear.concat", "operands": [ "s:230", "s:166" ], "params": {}, "parent": null, "results": [ "s:395" ] }, { "annotation": null, "id": "r:374", "op": "linear.slice_columns", "operands": [ "s:397" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:396" ] }, { "annotation": null, "id": "r:375", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:398" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:397" ] }, { "annotation": null, "id": "r:376", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:398" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 66 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:375", "results": [ "s:397" ] }, { "annotation": null, "id": "r:377", "op": "linear.concat", "operands": [ "s:297", "s:314" ], "params": {}, "parent": null, "results": [ "s:398" ] }, { "annotation": null, "id": "r:378", "op": "linear.slice_columns", "operands": [ "s:348" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:399" ] }, { "annotation": null, "id": "r:379", "op": "linear.slice_columns", "operands": [ "s:401" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:400" ] }, { "annotation": null, "id": "r:380", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:402" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:401" ] }, { "annotation": null, "id": "r:381", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:402" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 66 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:380", "results": [ "s:401" ] }, { "annotation": null, "id": "r:382", "op": "linear.concat", "operands": [ "s:101", "s:246" ], "params": {}, "parent": null, "results": [ "s:402" ] }, { "annotation": null, "id": "r:383", "op": "linear.slice_columns", "operands": [ "s:351" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:403" ] }, { "annotation": null, "id": "r:384", "op": "linear.add_const", "operands": [ "s:390" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:404" ] }, { "annotation": null, "id": "r:385", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:407", "s:413" ], "params": {}, "parent": null, "results": [ "s:405" ] }, { "annotation": null, "id": "r:386", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:406" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:385", "results": [ "s:405" ] }, { "annotation": null, "id": "r:387", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:410", "s:411" ], "params": {}, "parent": null, "results": [ "s:407" ] }, { "annotation": null, "id": "r:388", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:408" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:387", "results": [ "s:407" ] }, { "annotation": null, "id": "r:389", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:408" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:388", "results": [ "s:407" ] }, { "annotation": null, "id": "r:390", "op": "linear.sum_nodes", "operands": [ "s:410", "s:411" ], "params": { "max_fanout": null }, "parent": "r:387", "results": [ "s:408" ] }, { "annotation": null, "id": "r:391", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 3.5, "true_level": 1.0 }, "parent": null, "results": [ "s:410" ] }, { "annotation": null, "id": "r:392", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:391", "results": [ "s:410" ] }, { "annotation": null, "id": "r:393", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:412" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -4.5, "true_level": 1.0 }, "parent": null, "results": [ "s:411" ] }, { "annotation": null, "id": "r:394", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:412" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:393", "results": [ "s:411" ] }, { "annotation": null, "id": "r:395", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:412" ] }, { "annotation": null, "id": "r:396", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:417", "s:427", "s:449" ], "params": {}, "parent": null, "results": [ "s:413" ] }, { "annotation": null, "id": "r:397", "op": "linear.sum_nodes", "operands": [ "s:415", "s:425", "s:447" ], "params": { "max_fanout": null }, "parent": "r:396", "results": [ "s:413" ] }, { "annotation": null, "id": "r:398", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:417" ], "params": {}, "parent": "r:396", "results": [ "s:415" ] }, { "annotation": null, "id": "r:399", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:416" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:398", "results": [ "s:415" ] }, { "annotation": null, "id": "r:400", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:418" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:417" ] }, { "annotation": null, "id": "r:401", "op": "linear.bool_to_01", "operands": [ "s:420" ], "params": {}, "parent": null, "results": [ "s:418" ] }, { "annotation": null, "id": "r:402", "op": "linear.multiply_const", "operands": [ "s:419" ], "params": { "scalar": 0.5 }, "parent": "r:401", "results": [ "s:418" ] }, { "annotation": null, "id": "r:403", "op": "linear.add_const", "operands": [ "s:420" ], "params": { "scalar": 1.0 }, "parent": "r:401", "results": [ "s:419" ] }, { "annotation": null, "id": "r:404", "op": "swiglu.map_select.in_range", "operands": [ "s:422", "s:424" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:420" ] }, { "annotation": null, "id": "r:405", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:421" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:404", "results": [ "s:420" ] }, { "annotation": null, "id": "r:406", "op": "linear.add", "operands": [ "s:423", "s:330" ], "params": {}, "parent": null, "results": [ "s:422" ] }, { "annotation": null, "id": "r:407", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:423" ] }, { "annotation": null, "id": "r:408", "op": "linear.add_const", "operands": [ "s:422" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:424" ] }, { "annotation": null, "id": "r:409", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:427" ], "params": {}, "parent": "r:396", "results": [ "s:425" ] }, { "annotation": null, "id": "r:410", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:426" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:409", "results": [ "s:425" ] }, { "annotation": null, "id": "r:411", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:428" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:427" ] }, { "annotation": null, "id": "r:412", "op": "linear.bool_to_01", "operands": [ "s:430" ], "params": {}, "parent": null, "results": [ "s:428" ] }, { "annotation": null, "id": "r:413", "op": "linear.multiply_const", "operands": [ "s:429" ], "params": { "scalar": 0.5 }, "parent": "r:412", "results": [ "s:428" ] }, { "annotation": null, "id": "r:414", "op": "linear.add_const", "operands": [ "s:430" ], "params": { "scalar": 1.0 }, "parent": "r:412", "results": [ "s:429" ] }, { "annotation": null, "id": "r:415", "op": "swiglu.map_select.in_range", "operands": [ "s:432", "s:446" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:430" ] }, { "annotation": null, "id": "r:416", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:431" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:415", "results": [ "s:430" ] }, { "annotation": null, "id": "r:417", "op": "linear.add_const", "operands": [ "s:433" ], "params": { "scalar": 10.0 }, "parent": null, "results": [ "s:432" ] }, { "annotation": null, "id": "r:418", "op": "linear.subtract", "operands": [ "s:434", "s:445" ], "params": {}, "parent": null, "results": [ "s:433" ] }, { "annotation": null, "id": "r:419", "op": "linear.add", "operands": [ "s:434", "s:444" ], "params": {}, "parent": "r:418", "results": [ "s:433" ] }, { "annotation": null, "id": "r:420", "op": "linear.subtract", "operands": [ "s:435", "s:442" ], "params": {}, "parent": null, "results": [ "s:434" ] }, { "annotation": null, "id": "r:421", "op": "linear.add", "operands": [ "s:435", "s:441" ], "params": {}, "parent": "r:420", "results": [ "s:434" ] }, { "annotation": null, "id": "r:422", "op": "swiglu.map_select.select", "operands": [ "s:437", "s:439", "s:440" ], "params": {}, "parent": null, "results": [ "s:435" ] }, { "annotation": null, "id": "r:423", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:436" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } } }, "parent": "r:422", "results": [ "s:435" ] }, { "annotation": null, "id": "r:424", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:438" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": null, "results": [ "s:437" ] }, { "annotation": null, "id": "r:425", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:438" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:424", "results": [ "s:437" ] }, { "annotation": null, "id": "r:426", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:438" ] }, { "annotation": null, "id": "r:427", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:101" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:439" ] }, { "annotation": null, "id": "r:428", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:166" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:440" ] }, { "annotation": null, "id": "r:429", "op": "linear.negate", "operands": [ "s:442" ], "params": {}, "parent": "r:420", "results": [ "s:441" ] }, { "annotation": null, "id": "r:430", "op": "swiglu.map_select.select", "operands": [ "s:437", "s:440", "s:439" ], "params": {}, "parent": null, "results": [ "s:442" ] }, { "annotation": null, "id": "r:431", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:443" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } } }, "parent": "r:430", "results": [ "s:442" ] }, { "annotation": null, "id": "r:432", "op": "linear.negate", "operands": [ "s:445" ], "params": {}, "parent": "r:418", "results": [ "s:444" ] }, { "annotation": null, "id": "r:433", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:445" ] }, { "annotation": null, "id": "r:434", "op": "linear.add_const", "operands": [ "s:432" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:446" ] }, { "annotation": null, "id": "r:435", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:449" ], "params": {}, "parent": "r:396", "results": [ "s:447" ] }, { "annotation": null, "id": "r:436", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:448" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:435", "results": [ "s:447" ] }, { "annotation": null, "id": "r:437", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:450" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:449" ] }, { "annotation": null, "id": "r:438", "op": "linear.bool_to_01", "operands": [ "s:452" ], "params": {}, "parent": null, "results": [ "s:450" ] }, { "annotation": null, "id": "r:439", "op": "linear.multiply_const", "operands": [ "s:451" ], "params": { "scalar": 0.5 }, "parent": "r:438", "results": [ "s:450" ] }, { "annotation": null, "id": "r:440", "op": "linear.add_const", "operands": [ "s:452" ], "params": { "scalar": 1.0 }, "parent": "r:438", "results": [ "s:451" ] }, { "annotation": null, "id": "r:441", "op": "swiglu.map_select.in_range", "operands": [ "s:454", "s:466" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:452" ] }, { "annotation": null, "id": "r:442", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:453" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:441", "results": [ "s:452" ] }, { "annotation": null, "id": "r:443", "op": "linear.add", "operands": [ "s:455", "s:354" ], "params": {}, "parent": null, "results": [ "s:454" ] }, { "annotation": null, "id": "r:444", "op": "linear.sum_nodes", "operands": [ "s:457", "s:460", "s:461", "s:464", "s:465" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:455" ] }, { "annotation": null, "id": "r:445", "op": "linear.slice_columns", "operands": [ "s:458" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:457" ] }, { "annotation": null, "id": "r:446", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:459" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:458" ] }, { "annotation": null, "id": "r:447", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:459" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 66 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:446", "results": [ "s:458" ] }, { "annotation": null, "id": "r:448", "op": "linear.concat", "operands": [ "s:230", "s:314" ], "params": {}, "parent": null, "results": [ "s:459" ] }, { "annotation": null, "id": "r:449", "op": "linear.slice_columns", "operands": [ "s:394" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:460" ] }, { "annotation": null, "id": "r:450", "op": "linear.slice_columns", "operands": [ "s:462" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:461" ] }, { "annotation": null, "id": "r:451", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:463" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:462" ] }, { "annotation": null, "id": "r:452", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:463" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 66 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:451", "results": [ "s:462" ] }, { "annotation": null, "id": "r:453", "op": "linear.concat", "operands": [ "s:297", "s:246" ], "params": {}, "parent": null, "results": [ "s:463" ] }, { "annotation": null, "id": "r:454", "op": "linear.slice_columns", "operands": [ "s:397" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:464" ] }, { "annotation": null, "id": "r:455", "op": "linear.slice_columns", "operands": [ "s:401" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:465" ] }, { "annotation": null, "id": "r:456", "op": "linear.add_const", "operands": [ "s:454" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:466" ] }, { "annotation": null, "id": "r:457", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:469", "s:475" ], "params": {}, "parent": null, "results": [ "s:467" ] }, { "annotation": null, "id": "r:458", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:468" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:457", "results": [ "s:467" ] }, { "annotation": null, "id": "r:459", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:472", "s:473" ], "params": {}, "parent": null, "results": [ "s:469" ] }, { "annotation": null, "id": "r:460", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:470" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:459", "results": [ "s:469" ] }, { "annotation": null, "id": "r:461", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:470" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:460", "results": [ "s:469" ] }, { "annotation": null, "id": "r:462", "op": "linear.sum_nodes", "operands": [ "s:472", "s:473" ], "params": { "max_fanout": null }, "parent": "r:459", "results": [ "s:470" ] }, { "annotation": null, "id": "r:463", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 4.5, "true_level": 1.0 }, "parent": null, "results": [ "s:472" ] }, { "annotation": null, "id": "r:464", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:463", "results": [ "s:472" ] }, { "annotation": null, "id": "r:465", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:474" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -5.5, "true_level": 1.0 }, "parent": null, "results": [ "s:473" ] }, { "annotation": null, "id": "r:466", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:474" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:465", "results": [ "s:473" ] }, { "annotation": null, "id": "r:467", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:474" ] }, { "annotation": null, "id": "r:468", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:479", "s:491", "s:515" ], "params": {}, "parent": null, "results": [ "s:475" ] }, { "annotation": null, "id": "r:469", "op": "linear.sum_nodes", "operands": [ "s:477", "s:489", "s:513" ], "params": { "max_fanout": null }, "parent": "r:468", "results": [ "s:475" ] }, { "annotation": null, "id": "r:470", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:479" ], "params": {}, "parent": "r:468", "results": [ "s:477" ] }, { "annotation": null, "id": "r:471", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:478" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:470", "results": [ "s:477" ] }, { "annotation": null, "id": "r:472", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:480" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:479" ] }, { "annotation": null, "id": "r:473", "op": "linear.bool_to_01", "operands": [ "s:482" ], "params": {}, "parent": null, "results": [ "s:480" ] }, { "annotation": null, "id": "r:474", "op": "linear.multiply_const", "operands": [ "s:481" ], "params": { "scalar": 0.5 }, "parent": "r:473", "results": [ "s:480" ] }, { "annotation": null, "id": "r:475", "op": "linear.add_const", "operands": [ "s:482" ], "params": { "scalar": 1.0 }, "parent": "r:473", "results": [ "s:481" ] }, { "annotation": null, "id": "r:476", "op": "swiglu.map_select.in_range", "operands": [ "s:484", "s:488" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:482" ] }, { "annotation": null, "id": "r:477", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:483" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:476", "results": [ "s:482" ] }, { "annotation": null, "id": "r:478", "op": "linear.add", "operands": [ "s:423", "s:485" ], "params": {}, "parent": null, "results": [ "s:484" ] }, { "annotation": null, "id": "r:479", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:486" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:485" ] }, { "annotation": null, "id": "r:480", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -1, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:486" ] }, { "annotation": null, "id": "r:481", "op": "linear.add_const", "operands": [ "s:484" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:488" ] }, { "annotation": null, "id": "r:482", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:491" ], "params": {}, "parent": "r:468", "results": [ "s:489" ] }, { "annotation": null, "id": "r:483", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:490" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:482", "results": [ "s:489" ] }, { "annotation": null, "id": "r:484", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:492" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:491" ] }, { "annotation": null, "id": "r:485", "op": "linear.bool_to_01", "operands": [ "s:494" ], "params": {}, "parent": null, "results": [ "s:492" ] }, { "annotation": null, "id": "r:486", "op": "linear.multiply_const", "operands": [ "s:493" ], "params": { "scalar": 0.5 }, "parent": "r:485", "results": [ "s:492" ] }, { "annotation": null, "id": "r:487", "op": "linear.add_const", "operands": [ "s:494" ], "params": { "scalar": 1.0 }, "parent": "r:485", "results": [ "s:493" ] }, { "annotation": null, "id": "r:488", "op": "swiglu.map_select.in_range", "operands": [ "s:496", "s:512" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:494" ] }, { "annotation": null, "id": "r:489", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:495" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:488", "results": [ "s:494" ] }, { "annotation": null, "id": "r:490", "op": "linear.add_const", "operands": [ "s:497" ], "params": { "scalar": 10.0 }, "parent": null, "results": [ "s:496" ] }, { "annotation": null, "id": "r:491", "op": "linear.subtract", "operands": [ "s:498", "s:511" ], "params": {}, "parent": null, "results": [ "s:497" ] }, { "annotation": null, "id": "r:492", "op": "linear.add", "operands": [ "s:498", "s:510" ], "params": {}, "parent": "r:491", "results": [ "s:497" ] }, { "annotation": null, "id": "r:493", "op": "linear.subtract", "operands": [ "s:499", "s:508" ], "params": {}, "parent": null, "results": [ "s:498" ] }, { "annotation": null, "id": "r:494", "op": "linear.add", "operands": [ "s:499", "s:507" ], "params": {}, "parent": "r:493", "results": [ "s:498" ] }, { "annotation": null, "id": "r:495", "op": "swiglu.map_select.select", "operands": [ "s:501", "s:505", "s:506" ], "params": {}, "parent": null, "results": [ "s:499" ] }, { "annotation": null, "id": "r:496", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:500" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } } }, "parent": "r:495", "results": [ "s:499" ] }, { "annotation": null, "id": "r:497", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:502" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": null, "results": [ "s:501" ] }, { "annotation": null, "id": "r:498", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:502" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:497", "results": [ "s:501" ] }, { "annotation": null, "id": "r:499", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:503" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:502" ] }, { "annotation": null, "id": "r:500", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -1, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:503" ] }, { "annotation": null, "id": "r:501", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:297" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:505" ] }, { "annotation": null, "id": "r:502", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:314" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:506" ] }, { "annotation": null, "id": "r:503", "op": "linear.negate", "operands": [ "s:508" ], "params": {}, "parent": "r:493", "results": [ "s:507" ] }, { "annotation": null, "id": "r:504", "op": "swiglu.map_select.select", "operands": [ "s:501", "s:506", "s:505" ], "params": {}, "parent": null, "results": [ "s:508" ] }, { "annotation": null, "id": "r:505", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:509" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } } }, "parent": "r:504", "results": [ "s:508" ] }, { "annotation": null, "id": "r:506", "op": "linear.negate", "operands": [ "s:511" ], "params": {}, "parent": "r:491", "results": [ "s:510" ] }, { "annotation": null, "id": "r:507", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:511" ] }, { "annotation": null, "id": "r:508", "op": "linear.add_const", "operands": [ "s:496" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:512" ] }, { "annotation": null, "id": "r:509", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:515" ], "params": {}, "parent": "r:468", "results": [ "s:513" ] }, { "annotation": null, "id": "r:510", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:514" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:509", "results": [ "s:513" ] }, { "annotation": null, "id": "r:511", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:516" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:515" ] }, { "annotation": null, "id": "r:512", "op": "linear.bool_to_01", "operands": [ "s:518" ], "params": {}, "parent": null, "results": [ "s:516" ] }, { "annotation": null, "id": "r:513", "op": "linear.multiply_const", "operands": [ "s:517" ], "params": { "scalar": 0.5 }, "parent": "r:512", "results": [ "s:516" ] }, { "annotation": null, "id": "r:514", "op": "linear.add_const", "operands": [ "s:518" ], "params": { "scalar": 1.0 }, "parent": "r:512", "results": [ "s:517" ] }, { "annotation": null, "id": "r:515", "op": "swiglu.map_select.in_range", "operands": [ "s:520", "s:528" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:518" ] }, { "annotation": null, "id": "r:516", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:519" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:515", "results": [ "s:518" ] }, { "annotation": null, "id": "r:517", "op": "linear.add", "operands": [ "s:521", "s:354" ], "params": {}, "parent": null, "results": [ "s:520" ] }, { "annotation": null, "id": "r:518", "op": "linear.sum_nodes", "operands": [ "s:523", "s:526", "s:527" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:521" ] }, { "annotation": null, "id": "r:519", "op": "linear.slice_columns", "operands": [ "s:524" ], "params": { "name": "product_ones", "start": 1, "width": 1 }, "parent": null, "results": [ "s:523" ] }, { "annotation": null, "id": "r:520", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:525" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:524" ] }, { "annotation": null, "id": "r:521", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:525" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 66 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 2 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:520", "results": [ "s:524" ] }, { "annotation": null, "id": "r:522", "op": "linear.concat", "operands": [ "s:230", "s:246" ], "params": {}, "parent": null, "results": [ "s:525" ] }, { "annotation": null, "id": "r:523", "op": "linear.slice_columns", "operands": [ "s:458" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:526" ] }, { "annotation": null, "id": "r:524", "op": "linear.slice_columns", "operands": [ "s:462" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:527" ] }, { "annotation": null, "id": "r:525", "op": "linear.add_const", "operands": [ "s:520" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:528" ] }, { "annotation": null, "id": "r:526", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:531", "s:537" ], "params": {}, "parent": null, "results": [ "s:529" ] }, { "annotation": null, "id": "r:527", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:530" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:526", "results": [ "s:529" ] }, { "annotation": null, "id": "r:528", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:534", "s:535" ], "params": {}, "parent": null, "results": [ "s:531" ] }, { "annotation": null, "id": "r:529", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:532" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:528", "results": [ "s:531" ] }, { "annotation": null, "id": "r:530", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:532" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:529", "results": [ "s:531" ] }, { "annotation": null, "id": "r:531", "op": "linear.sum_nodes", "operands": [ "s:534", "s:535" ], "params": { "max_fanout": null }, "parent": "r:528", "results": [ "s:532" ] }, { "annotation": null, "id": "r:532", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 5.5, "true_level": 1.0 }, "parent": null, "results": [ "s:534" ] }, { "annotation": null, "id": "r:533", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:532", "results": [ "s:534" ] }, { "annotation": null, "id": "r:534", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:536" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -6.5, "true_level": 1.0 }, "parent": null, "results": [ "s:535" ] }, { "annotation": null, "id": "r:535", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:536" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:534", "results": [ "s:535" ] }, { "annotation": null, "id": "r:536", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:536" ] }, { "annotation": null, "id": "r:537", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:541", "s:553", "s:576" ], "params": {}, "parent": null, "results": [ "s:537" ] }, { "annotation": null, "id": "r:538", "op": "linear.sum_nodes", "operands": [ "s:539", "s:551", "s:574" ], "params": { "max_fanout": null }, "parent": "r:537", "results": [ "s:537" ] }, { "annotation": null, "id": "r:539", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:541" ], "params": {}, "parent": "r:537", "results": [ "s:539" ] }, { "annotation": null, "id": "r:540", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:540" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:539", "results": [ "s:539" ] }, { "annotation": null, "id": "r:541", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:542" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:541" ] }, { "annotation": null, "id": "r:542", "op": "linear.bool_to_01", "operands": [ "s:544" ], "params": {}, "parent": null, "results": [ "s:542" ] }, { "annotation": null, "id": "r:543", "op": "linear.multiply_const", "operands": [ "s:543" ], "params": { "scalar": 0.5 }, "parent": "r:542", "results": [ "s:542" ] }, { "annotation": null, "id": "r:544", "op": "linear.add_const", "operands": [ "s:544" ], "params": { "scalar": 1.0 }, "parent": "r:542", "results": [ "s:543" ] }, { "annotation": null, "id": "r:545", "op": "swiglu.map_select.in_range", "operands": [ "s:546", "s:550" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:544" ] }, { "annotation": null, "id": "r:546", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:545" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:545", "results": [ "s:544" ] }, { "annotation": null, "id": "r:547", "op": "linear.add", "operands": [ "s:374", "s:547" ], "params": {}, "parent": null, "results": [ "s:546" ] }, { "annotation": null, "id": "r:548", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:548" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:547" ] }, { "annotation": null, "id": "r:549", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -3, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:548" ] }, { "annotation": null, "id": "r:550", "op": "linear.add_const", "operands": [ "s:546" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:550" ] }, { "annotation": null, "id": "r:551", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:553" ], "params": {}, "parent": "r:537", "results": [ "s:551" ] }, { "annotation": null, "id": "r:552", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:552" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:551", "results": [ "s:551" ] }, { "annotation": null, "id": "r:553", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:554" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:553" ] }, { "annotation": null, "id": "r:554", "op": "linear.bool_to_01", "operands": [ "s:556" ], "params": {}, "parent": null, "results": [ "s:554" ] }, { "annotation": null, "id": "r:555", "op": "linear.multiply_const", "operands": [ "s:555" ], "params": { "scalar": 0.5 }, "parent": "r:554", "results": [ "s:554" ] }, { "annotation": null, "id": "r:556", "op": "linear.add_const", "operands": [ "s:556" ], "params": { "scalar": 1.0 }, "parent": "r:554", "results": [ "s:555" ] }, { "annotation": null, "id": "r:557", "op": "swiglu.map_select.in_range", "operands": [ "s:558", "s:573" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:556" ] }, { "annotation": null, "id": "r:558", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:557" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:557", "results": [ "s:556" ] }, { "annotation": null, "id": "r:559", "op": "linear.add_const", "operands": [ "s:559" ], "params": { "scalar": 10.0 }, "parent": null, "results": [ "s:558" ] }, { "annotation": null, "id": "r:560", "op": "linear.subtract", "operands": [ "s:560", "s:511" ], "params": {}, "parent": null, "results": [ "s:559" ] }, { "annotation": null, "id": "r:561", "op": "linear.add", "operands": [ "s:560", "s:572" ], "params": {}, "parent": "r:560", "results": [ "s:559" ] }, { "annotation": null, "id": "r:562", "op": "linear.subtract", "operands": [ "s:561", "s:570" ], "params": {}, "parent": null, "results": [ "s:560" ] }, { "annotation": null, "id": "r:563", "op": "linear.add", "operands": [ "s:561", "s:569" ], "params": {}, "parent": "r:562", "results": [ "s:560" ] }, { "annotation": null, "id": "r:564", "op": "swiglu.map_select.select", "operands": [ "s:563", "s:567", "s:568" ], "params": {}, "parent": null, "results": [ "s:561" ] }, { "annotation": null, "id": "r:565", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:562" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } } }, "parent": "r:564", "results": [ "s:561" ] }, { "annotation": null, "id": "r:566", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:564" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": null, "results": [ "s:563" ] }, { "annotation": null, "id": "r:567", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:564" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:566", "results": [ "s:563" ] }, { "annotation": null, "id": "r:568", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:565" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:564" ] }, { "annotation": null, "id": "r:569", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -2, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:565" ] }, { "annotation": null, "id": "r:570", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:230" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:567" ] }, { "annotation": null, "id": "r:571", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:246" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:568" ] }, { "annotation": null, "id": "r:572", "op": "linear.negate", "operands": [ "s:570" ], "params": {}, "parent": "r:562", "results": [ "s:569" ] }, { "annotation": null, "id": "r:573", "op": "swiglu.map_select.select", "operands": [ "s:563", "s:568", "s:567" ], "params": {}, "parent": null, "results": [ "s:570" ] }, { "annotation": null, "id": "r:574", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:571" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } } }, "parent": "r:573", "results": [ "s:570" ] }, { "annotation": null, "id": "r:575", "op": "linear.negate", "operands": [ "s:511" ], "params": {}, "parent": "r:560", "results": [ "s:572" ] }, { "annotation": null, "id": "r:576", "op": "linear.add_const", "operands": [ "s:558" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:573" ] }, { "annotation": null, "id": "r:577", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:576" ], "params": {}, "parent": "r:537", "results": [ "s:574" ] }, { "annotation": null, "id": "r:578", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:575" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:577", "results": [ "s:574" ] }, { "annotation": null, "id": "r:579", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:577" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:576" ] }, { "annotation": null, "id": "r:580", "op": "linear.bool_to_01", "operands": [ "s:579" ], "params": {}, "parent": null, "results": [ "s:577" ] }, { "annotation": null, "id": "r:581", "op": "linear.multiply_const", "operands": [ "s:578" ], "params": { "scalar": 0.5 }, "parent": "r:580", "results": [ "s:577" ] }, { "annotation": null, "id": "r:582", "op": "linear.add_const", "operands": [ "s:579" ], "params": { "scalar": 1.0 }, "parent": "r:580", "results": [ "s:578" ] }, { "annotation": null, "id": "r:583", "op": "swiglu.map_select.in_range", "operands": [ "s:581", "s:585" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:579" ] }, { "annotation": null, "id": "r:584", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:580" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:583", "results": [ "s:579" ] }, { "annotation": null, "id": "r:585", "op": "linear.add", "operands": [ "s:582", "s:354" ], "params": {}, "parent": null, "results": [ "s:581" ] }, { "annotation": null, "id": "r:586", "op": "linear.sum_nodes", "operands": [ "s:584" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:582" ] }, { "annotation": null, "id": "r:587", "op": "linear.slice_columns", "operands": [ "s:524" ], "params": { "name": "product_tens", "start": 0, "width": 1 }, "parent": null, "results": [ "s:584" ] }, { "annotation": null, "id": "r:588", "op": "linear.add_const", "operands": [ "s:581" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:585" ] }, { "annotation": null, "id": "r:589", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:588", "s:594" ], "params": {}, "parent": null, "results": [ "s:586" ] }, { "annotation": null, "id": "r:590", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:587" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:589", "results": [ "s:586" ] }, { "annotation": null, "id": "r:591", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:591", "s:592" ], "params": {}, "parent": null, "results": [ "s:588" ] }, { "annotation": null, "id": "r:592", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:589" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:591", "results": [ "s:588" ] }, { "annotation": null, "id": "r:593", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:589" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:592", "results": [ "s:588" ] }, { "annotation": null, "id": "r:594", "op": "linear.sum_nodes", "operands": [ "s:591", "s:592" ], "params": { "max_fanout": null }, "parent": "r:591", "results": [ "s:589" ] }, { "annotation": null, "id": "r:595", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 6.5, "true_level": 1.0 }, "parent": null, "results": [ "s:591" ] }, { "annotation": null, "id": "r:596", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:595", "results": [ "s:591" ] }, { "annotation": null, "id": "r:597", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:593" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -7.5, "true_level": 1.0 }, "parent": null, "results": [ "s:592" ] }, { "annotation": null, "id": "r:598", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:593" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:597", "results": [ "s:592" ] }, { "annotation": null, "id": "r:599", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:593" ] }, { "annotation": null, "id": "r:600", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:598", "s:610", "s:634" ], "params": {}, "parent": null, "results": [ "s:594" ] }, { "annotation": null, "id": "r:601", "op": "linear.sum_nodes", "operands": [ "s:596", "s:608", "s:632" ], "params": { "max_fanout": null }, "parent": "r:600", "results": [ "s:594" ] }, { "annotation": null, "id": "r:602", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:598" ], "params": {}, "parent": "r:600", "results": [ "s:596" ] }, { "annotation": null, "id": "r:603", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:597" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:602", "results": [ "s:596" ] }, { "annotation": null, "id": "r:604", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:599" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:598" ] }, { "annotation": null, "id": "r:605", "op": "linear.bool_to_01", "operands": [ "s:601" ], "params": {}, "parent": null, "results": [ "s:599" ] }, { "annotation": null, "id": "r:606", "op": "linear.multiply_const", "operands": [ "s:600" ], "params": { "scalar": 0.5 }, "parent": "r:605", "results": [ "s:599" ] }, { "annotation": null, "id": "r:607", "op": "linear.add_const", "operands": [ "s:601" ], "params": { "scalar": 1.0 }, "parent": "r:605", "results": [ "s:600" ] }, { "annotation": null, "id": "r:608", "op": "swiglu.map_select.in_range", "operands": [ "s:603", "s:607" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:601" ] }, { "annotation": null, "id": "r:609", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:602" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:608", "results": [ "s:601" ] }, { "annotation": null, "id": "r:610", "op": "linear.add", "operands": [ "s:295", "s:604" ], "params": {}, "parent": null, "results": [ "s:603" ] }, { "annotation": null, "id": "r:611", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:605" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:604" ] }, { "annotation": null, "id": "r:612", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -5, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:605" ] }, { "annotation": null, "id": "r:613", "op": "linear.add_const", "operands": [ "s:603" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:607" ] }, { "annotation": null, "id": "r:614", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:610" ], "params": {}, "parent": "r:600", "results": [ "s:608" ] }, { "annotation": null, "id": "r:615", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:609" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:614", "results": [ "s:608" ] }, { "annotation": null, "id": "r:616", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:611" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:610" ] }, { "annotation": null, "id": "r:617", "op": "linear.bool_to_01", "operands": [ "s:613" ], "params": {}, "parent": null, "results": [ "s:611" ] }, { "annotation": null, "id": "r:618", "op": "linear.multiply_const", "operands": [ "s:612" ], "params": { "scalar": 0.5 }, "parent": "r:617", "results": [ "s:611" ] }, { "annotation": null, "id": "r:619", "op": "linear.add_const", "operands": [ "s:613" ], "params": { "scalar": 1.0 }, "parent": "r:617", "results": [ "s:612" ] }, { "annotation": null, "id": "r:620", "op": "swiglu.map_select.in_range", "operands": [ "s:615", "s:631" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:613" ] }, { "annotation": null, "id": "r:621", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:614" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:620", "results": [ "s:613" ] }, { "annotation": null, "id": "r:622", "op": "linear.add_const", "operands": [ "s:616" ], "params": { "scalar": 10.0 }, "parent": null, "results": [ "s:615" ] }, { "annotation": null, "id": "r:623", "op": "linear.subtract", "operands": [ "s:617", "s:628" ], "params": {}, "parent": null, "results": [ "s:616" ] }, { "annotation": null, "id": "r:624", "op": "linear.add", "operands": [ "s:617", "s:627" ], "params": {}, "parent": "r:623", "results": [ "s:616" ] }, { "annotation": null, "id": "r:625", "op": "linear.subtract", "operands": [ "s:618", "s:625" ], "params": {}, "parent": null, "results": [ "s:617" ] }, { "annotation": null, "id": "r:626", "op": "linear.add", "operands": [ "s:618", "s:624" ], "params": {}, "parent": "r:625", "results": [ "s:617" ] }, { "annotation": null, "id": "r:627", "op": "swiglu.map_select.select", "operands": [ "s:620", "s:567", "s:568" ], "params": {}, "parent": null, "results": [ "s:618" ] }, { "annotation": null, "id": "r:628", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:619" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } } }, "parent": "r:627", "results": [ "s:618" ] }, { "annotation": null, "id": "r:629", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:621" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": null, "results": [ "s:620" ] }, { "annotation": null, "id": "r:630", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:621" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:629", "results": [ "s:620" ] }, { "annotation": null, "id": "r:631", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:622" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:621" ] }, { "annotation": null, "id": "r:632", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -3, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:622" ] }, { "annotation": null, "id": "r:633", "op": "linear.negate", "operands": [ "s:625" ], "params": {}, "parent": "r:625", "results": [ "s:624" ] }, { "annotation": null, "id": "r:634", "op": "swiglu.map_select.select", "operands": [ "s:620", "s:568", "s:567" ], "params": {}, "parent": null, "results": [ "s:625" ] }, { "annotation": null, "id": "r:635", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:626" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } } }, "parent": "r:634", "results": [ "s:625" ] }, { "annotation": null, "id": "r:636", "op": "linear.negate", "operands": [ "s:628" ], "params": {}, "parent": "r:623", "results": [ "s:627" ] }, { "annotation": null, "id": "r:637", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:629" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:628" ] }, { "annotation": null, "id": "r:638", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -1, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:629" ] }, { "annotation": null, "id": "r:639", "op": "linear.add_const", "operands": [ "s:615" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:631" ] }, { "annotation": null, "id": "r:640", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:634" ], "params": {}, "parent": "r:600", "results": [ "s:632" ] }, { "annotation": null, "id": "r:641", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:633" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:640", "results": [ "s:632" ] }, { "annotation": null, "id": "r:642", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:635" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:634" ] }, { "annotation": null, "id": "r:643", "op": "linear.bool_to_01", "operands": [ "s:637" ], "params": {}, "parent": null, "results": [ "s:635" ] }, { "annotation": null, "id": "r:644", "op": "linear.multiply_const", "operands": [ "s:636" ], "params": { "scalar": 0.5 }, "parent": "r:643", "results": [ "s:635" ] }, { "annotation": null, "id": "r:645", "op": "linear.add_const", "operands": [ "s:637" ], "params": { "scalar": 1.0 }, "parent": "r:643", "results": [ "s:636" ] }, { "annotation": null, "id": "r:646", "op": "swiglu.map_select.in_range", "operands": [ "s:639", "s:643" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:637" ] }, { "annotation": null, "id": "r:647", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:638" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:646", "results": [ "s:637" ] }, { "annotation": null, "id": "r:648", "op": "linear.add", "operands": [ "s:582", "s:640" ], "params": {}, "parent": null, "results": [ "s:639" ] }, { "annotation": null, "id": "r:649", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:641" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:640" ] }, { "annotation": null, "id": "r:650", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -1, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:641" ] }, { "annotation": null, "id": "r:651", "op": "linear.add_const", "operands": [ "s:639" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:643" ] }, { "annotation": null, "id": "r:652", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:646", "s:652" ], "params": {}, "parent": null, "results": [ "s:644" ] }, { "annotation": null, "id": "r:653", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:645" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:652", "results": [ "s:644" ] }, { "annotation": null, "id": "r:654", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:649", "s:650" ], "params": {}, "parent": null, "results": [ "s:646" ] }, { "annotation": null, "id": "r:655", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:647" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:654", "results": [ "s:646" ] }, { "annotation": null, "id": "r:656", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:647" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:655", "results": [ "s:646" ] }, { "annotation": null, "id": "r:657", "op": "linear.sum_nodes", "operands": [ "s:649", "s:650" ], "params": { "max_fanout": null }, "parent": "r:654", "results": [ "s:647" ] }, { "annotation": null, "id": "r:658", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 7.5, "true_level": 1.0 }, "parent": null, "results": [ "s:649" ] }, { "annotation": null, "id": "r:659", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:658", "results": [ "s:649" ] }, { "annotation": null, "id": "r:660", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:651" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -8.5, "true_level": 1.0 }, "parent": null, "results": [ "s:650" ] }, { "annotation": null, "id": "r:661", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:651" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:660", "results": [ "s:650" ] }, { "annotation": null, "id": "r:662", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:651" ] }, { "annotation": null, "id": "r:663", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:656", "s:666", "s:690" ], "params": {}, "parent": null, "results": [ "s:652" ] }, { "annotation": null, "id": "r:664", "op": "linear.sum_nodes", "operands": [ "s:654", "s:664", "s:688" ], "params": { "max_fanout": null }, "parent": "r:663", "results": [ "s:652" ] }, { "annotation": null, "id": "r:665", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:656" ], "params": {}, "parent": "r:663", "results": [ "s:654" ] }, { "annotation": null, "id": "r:666", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:655" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:665", "results": [ "s:654" ] }, { "annotation": null, "id": "r:667", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:657" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:656" ] }, { "annotation": null, "id": "r:668", "op": "linear.bool_to_01", "operands": [ "s:659" ], "params": {}, "parent": null, "results": [ "s:657" ] }, { "annotation": null, "id": "r:669", "op": "linear.multiply_const", "operands": [ "s:658" ], "params": { "scalar": 0.5 }, "parent": "r:668", "results": [ "s:657" ] }, { "annotation": null, "id": "r:670", "op": "linear.add_const", "operands": [ "s:659" ], "params": { "scalar": 1.0 }, "parent": "r:668", "results": [ "s:658" ] }, { "annotation": null, "id": "r:671", "op": "swiglu.map_select.in_range", "operands": [ "s:661", "s:663" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:659" ] }, { "annotation": null, "id": "r:672", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:660" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:671", "results": [ "s:659" ] }, { "annotation": null, "id": "r:673", "op": "linear.add", "operands": [ "s:99", "s:662" ], "params": {}, "parent": null, "results": [ "s:661" ] }, { "annotation": null, "id": "r:674", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:662" ] }, { "annotation": null, "id": "r:675", "op": "linear.add_const", "operands": [ "s:661" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:663" ] }, { "annotation": null, "id": "r:676", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:666" ], "params": {}, "parent": "r:663", "results": [ "s:664" ] }, { "annotation": null, "id": "r:677", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:665" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:676", "results": [ "s:664" ] }, { "annotation": null, "id": "r:678", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:667" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:666" ] }, { "annotation": null, "id": "r:679", "op": "linear.bool_to_01", "operands": [ "s:669" ], "params": {}, "parent": null, "results": [ "s:667" ] }, { "annotation": null, "id": "r:680", "op": "linear.multiply_const", "operands": [ "s:668" ], "params": { "scalar": 0.5 }, "parent": "r:679", "results": [ "s:667" ] }, { "annotation": null, "id": "r:681", "op": "linear.add_const", "operands": [ "s:669" ], "params": { "scalar": 1.0 }, "parent": "r:679", "results": [ "s:668" ] }, { "annotation": null, "id": "r:682", "op": "swiglu.map_select.in_range", "operands": [ "s:671", "s:687" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:669" ] }, { "annotation": null, "id": "r:683", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:670" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:682", "results": [ "s:669" ] }, { "annotation": null, "id": "r:684", "op": "linear.add_const", "operands": [ "s:672" ], "params": { "scalar": 10.0 }, "parent": null, "results": [ "s:671" ] }, { "annotation": null, "id": "r:685", "op": "linear.subtract", "operands": [ "s:673", "s:684" ], "params": {}, "parent": null, "results": [ "s:672" ] }, { "annotation": null, "id": "r:686", "op": "linear.add", "operands": [ "s:673", "s:683" ], "params": {}, "parent": "r:685", "results": [ "s:672" ] }, { "annotation": null, "id": "r:687", "op": "linear.subtract", "operands": [ "s:674", "s:681" ], "params": {}, "parent": null, "results": [ "s:673" ] }, { "annotation": null, "id": "r:688", "op": "linear.add", "operands": [ "s:674", "s:680" ], "params": {}, "parent": "r:687", "results": [ "s:673" ] }, { "annotation": null, "id": "r:689", "op": "swiglu.map_select.select", "operands": [ "s:676", "s:505", "s:506" ], "params": {}, "parent": null, "results": [ "s:674" ] }, { "annotation": null, "id": "r:690", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:675" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } } }, "parent": "r:689", "results": [ "s:674" ] }, { "annotation": null, "id": "r:691", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:677" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": null, "results": [ "s:676" ] }, { "annotation": null, "id": "r:692", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:677" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:691", "results": [ "s:676" ] }, { "annotation": null, "id": "r:693", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:678" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:677" ] }, { "annotation": null, "id": "r:694", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -4, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:678" ] }, { "annotation": null, "id": "r:695", "op": "linear.negate", "operands": [ "s:681" ], "params": {}, "parent": "r:687", "results": [ "s:680" ] }, { "annotation": null, "id": "r:696", "op": "swiglu.map_select.select", "operands": [ "s:676", "s:506", "s:505" ], "params": {}, "parent": null, "results": [ "s:681" ] }, { "annotation": null, "id": "r:697", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:682" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } } }, "parent": "r:696", "results": [ "s:681" ] }, { "annotation": null, "id": "r:698", "op": "linear.negate", "operands": [ "s:684" ], "params": {}, "parent": "r:685", "results": [ "s:683" ] }, { "annotation": null, "id": "r:699", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:685" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:684" ] }, { "annotation": null, "id": "r:700", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -3, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:685" ] }, { "annotation": null, "id": "r:701", "op": "linear.add_const", "operands": [ "s:671" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:687" ] }, { "annotation": null, "id": "r:702", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:690" ], "params": {}, "parent": "r:663", "results": [ "s:688" ] }, { "annotation": null, "id": "r:703", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:689" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:702", "results": [ "s:688" ] }, { "annotation": null, "id": "r:704", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:691" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:690" ] }, { "annotation": null, "id": "r:705", "op": "linear.bool_to_01", "operands": [ "s:693" ], "params": {}, "parent": null, "results": [ "s:691" ] }, { "annotation": null, "id": "r:706", "op": "linear.multiply_const", "operands": [ "s:692" ], "params": { "scalar": 0.5 }, "parent": "r:705", "results": [ "s:691" ] }, { "annotation": null, "id": "r:707", "op": "linear.add_const", "operands": [ "s:693" ], "params": { "scalar": 1.0 }, "parent": "r:705", "results": [ "s:692" ] }, { "annotation": null, "id": "r:708", "op": "swiglu.map_select.in_range", "operands": [ "s:695", "s:699" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:693" ] }, { "annotation": null, "id": "r:709", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:694" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:708", "results": [ "s:693" ] }, { "annotation": null, "id": "r:710", "op": "linear.add", "operands": [ "s:521", "s:696" ], "params": {}, "parent": null, "results": [ "s:695" ] }, { "annotation": null, "id": "r:711", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:697" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:696" ] }, { "annotation": null, "id": "r:712", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -3, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:697" ] }, { "annotation": null, "id": "r:713", "op": "linear.add_const", "operands": [ "s:695" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:699" ] }, { "annotation": null, "id": "r:714", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:702", "s:708" ], "params": {}, "parent": null, "results": [ "s:700" ] }, { "annotation": null, "id": "r:715", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:701" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:714", "results": [ "s:700" ] }, { "annotation": null, "id": "r:716", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:705", "s:706" ], "params": {}, "parent": null, "results": [ "s:702" ] }, { "annotation": null, "id": "r:717", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:703" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:716", "results": [ "s:702" ] }, { "annotation": null, "id": "r:718", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:703" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:717", "results": [ "s:702" ] }, { "annotation": null, "id": "r:719", "op": "linear.sum_nodes", "operands": [ "s:705", "s:706" ], "params": { "max_fanout": null }, "parent": "r:716", "results": [ "s:703" ] }, { "annotation": null, "id": "r:720", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 8.5, "true_level": 1.0 }, "parent": null, "results": [ "s:705" ] }, { "annotation": null, "id": "r:721", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:720", "results": [ "s:705" ] }, { "annotation": null, "id": "r:722", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:707" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -9.5, "true_level": 1.0 }, "parent": null, "results": [ "s:706" ] }, { "annotation": null, "id": "r:723", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:707" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:722", "results": [ "s:706" ] }, { "annotation": null, "id": "r:724", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:707" ] }, { "annotation": null, "id": "r:725", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:712", "s:731", "s:753" ], "params": {}, "parent": null, "results": [ "s:708" ] }, { "annotation": null, "id": "r:726", "op": "linear.sum_nodes", "operands": [ "s:710", "s:729", "s:751" ], "params": { "max_fanout": null }, "parent": "r:725", "results": [ "s:708" ] }, { "annotation": null, "id": "r:727", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:712" ], "params": {}, "parent": "r:725", "results": [ "s:710" ] }, { "annotation": null, "id": "r:728", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:711" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:727", "results": [ "s:710" ] }, { "annotation": null, "id": "r:729", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:713" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:712" ] }, { "annotation": null, "id": "r:730", "op": "linear.bool_to_01", "operands": [ "s:715" ], "params": {}, "parent": null, "results": [ "s:713" ] }, { "annotation": null, "id": "r:731", "op": "linear.multiply_const", "operands": [ "s:714" ], "params": { "scalar": 0.5 }, "parent": "r:730", "results": [ "s:713" ] }, { "annotation": null, "id": "r:732", "op": "linear.add_const", "operands": [ "s:715" ], "params": { "scalar": 1.0 }, "parent": "r:730", "results": [ "s:714" ] }, { "annotation": null, "id": "r:733", "op": "swiglu.map_select.in_range", "operands": [ "s:717", "s:728" ], "params": { "n_slots": 7 }, "parent": null, "results": [ "s:715" ] }, { "annotation": null, "id": "r:734", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:716" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 28 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 7 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 7 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:733", "results": [ "s:715" ] }, { "annotation": null, "id": "r:735", "op": "linear.add", "operands": [ "s:718", "s:719" ], "params": {}, "parent": null, "results": [ "s:717" ] }, { "annotation": null, "id": "r:736", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:718" ] }, { "annotation": null, "id": "r:737", "op": "linear.bool_to_01", "operands": [ "s:721" ], "params": {}, "parent": null, "results": [ "s:719" ] }, { "annotation": null, "id": "r:738", "op": "linear.multiply_const", "operands": [ "s:720" ], "params": { "scalar": 0.5 }, "parent": "r:737", "results": [ "s:719" ] }, { "annotation": null, "id": "r:739", "op": "linear.add_const", "operands": [ "s:721" ], "params": { "scalar": 1.0 }, "parent": "r:737", "results": [ "s:720" ] }, { "annotation": null, "id": "r:740", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:724", "s:725" ], "params": {}, "parent": null, "results": [ "s:721" ] }, { "annotation": null, "id": "r:741", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:722" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:740", "results": [ "s:721" ] }, { "annotation": null, "id": "r:742", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:722" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:741", "results": [ "s:721" ] }, { "annotation": null, "id": "r:743", "op": "linear.sum_nodes", "operands": [ "s:724", "s:725" ], "params": { "max_fanout": null }, "parent": "r:740", "results": [ "s:722" ] }, { "annotation": null, "id": "r:744", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:718" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:724" ] }, { "annotation": null, "id": "r:745", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:718" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:744", "results": [ "s:724" ] }, { "annotation": null, "id": "r:746", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:726" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:725" ] }, { "annotation": null, "id": "r:747", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:726" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:746", "results": [ "s:725" ] }, { "annotation": null, "id": "r:748", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -3, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:726" ] }, { "annotation": null, "id": "r:749", "op": "linear.add_const", "operands": [ "s:717" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:728" ] }, { "annotation": null, "id": "r:750", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:731" ], "params": {}, "parent": "r:725", "results": [ "s:729" ] }, { "annotation": null, "id": "r:751", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:730" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:750", "results": [ "s:729" ] }, { "annotation": null, "id": "r:752", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:732" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:731" ] }, { "annotation": null, "id": "r:753", "op": "linear.bool_to_01", "operands": [ "s:734" ], "params": {}, "parent": null, "results": [ "s:732" ] }, { "annotation": null, "id": "r:754", "op": "linear.multiply_const", "operands": [ "s:733" ], "params": { "scalar": 0.5 }, "parent": "r:753", "results": [ "s:732" ] }, { "annotation": null, "id": "r:755", "op": "linear.add_const", "operands": [ "s:734" ], "params": { "scalar": 1.0 }, "parent": "r:753", "results": [ "s:733" ] }, { "annotation": null, "id": "r:756", "op": "swiglu.map_select.in_range", "operands": [ "s:736", "s:750" ], "params": { "n_slots": 20 }, "parent": null, "results": [ "s:734" ] }, { "annotation": null, "id": "r:757", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:735" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 80 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 20 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 80, 20 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:756", "results": [ "s:734" ] }, { "annotation": null, "id": "r:758", "op": "linear.add_const", "operands": [ "s:737" ], "params": { "scalar": 10.0 }, "parent": null, "results": [ "s:736" ] }, { "annotation": null, "id": "r:759", "op": "linear.subtract", "operands": [ "s:738", "s:749" ], "params": {}, "parent": null, "results": [ "s:737" ] }, { "annotation": null, "id": "r:760", "op": "linear.add", "operands": [ "s:738", "s:748" ], "params": {}, "parent": "r:759", "results": [ "s:737" ] }, { "annotation": null, "id": "r:761", "op": "linear.subtract", "operands": [ "s:739", "s:746" ], "params": {}, "parent": null, "results": [ "s:738" ] }, { "annotation": null, "id": "r:762", "op": "linear.add", "operands": [ "s:739", "s:745" ], "params": {}, "parent": "r:761", "results": [ "s:738" ] }, { "annotation": null, "id": "r:763", "op": "swiglu.map_select.select", "operands": [ "s:741", "s:439", "s:440" ], "params": {}, "parent": null, "results": [ "s:739" ] }, { "annotation": null, "id": "r:764", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:740" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } } }, "parent": "r:763", "results": [ "s:739" ] }, { "annotation": null, "id": "r:765", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:742" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": null, "results": [ "s:741" ] }, { "annotation": null, "id": "r:766", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:742" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:765", "results": [ "s:741" ] }, { "annotation": null, "id": "r:767", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:743" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:742" ] }, { "annotation": null, "id": "r:768", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -5, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:743" ] }, { "annotation": null, "id": "r:769", "op": "linear.negate", "operands": [ "s:746" ], "params": {}, "parent": "r:761", "results": [ "s:745" ] }, { "annotation": null, "id": "r:770", "op": "swiglu.map_select.select", "operands": [ "s:741", "s:440", "s:439" ], "params": {}, "parent": null, "results": [ "s:746" ] }, { "annotation": null, "id": "r:771", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:747" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 3 ] } } }, "parent": "r:770", "results": [ "s:746" ] }, { "annotation": null, "id": "r:772", "op": "linear.negate", "operands": [ "s:749" ], "params": {}, "parent": "r:759", "results": [ "s:748" ] }, { "annotation": null, "id": "r:773", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:749" ] }, { "annotation": null, "id": "r:774", "op": "linear.add_const", "operands": [ "s:736" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:750" ] }, { "annotation": null, "id": "r:775", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:753" ], "params": {}, "parent": "r:725", "results": [ "s:751" ] }, { "annotation": null, "id": "r:776", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:752" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:775", "results": [ "s:751" ] }, { "annotation": null, "id": "r:777", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:754" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:753" ] }, { "annotation": null, "id": "r:778", "op": "linear.bool_to_01", "operands": [ "s:756" ], "params": {}, "parent": null, "results": [ "s:754" ] }, { "annotation": null, "id": "r:779", "op": "linear.multiply_const", "operands": [ "s:755" ], "params": { "scalar": 0.5 }, "parent": "r:778", "results": [ "s:754" ] }, { "annotation": null, "id": "r:780", "op": "linear.add_const", "operands": [ "s:756" ], "params": { "scalar": 1.0 }, "parent": "r:778", "results": [ "s:755" ] }, { "annotation": null, "id": "r:781", "op": "swiglu.map_select.in_range", "operands": [ "s:758", "s:762" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:756" ] }, { "annotation": null, "id": "r:782", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:757" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:781", "results": [ "s:756" ] }, { "annotation": null, "id": "r:783", "op": "linear.add", "operands": [ "s:455", "s:759" ], "params": {}, "parent": null, "results": [ "s:758" ] }, { "annotation": null, "id": "r:784", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:760" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:759" ] }, { "annotation": null, "id": "r:785", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -5, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:760" ] }, { "annotation": null, "id": "r:786", "op": "linear.add_const", "operands": [ "s:758" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:762" ] }, { "annotation": null, "id": "r:787", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:765", "s:771" ], "params": {}, "parent": null, "results": [ "s:763" ] }, { "annotation": null, "id": "r:788", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:764" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:787", "results": [ "s:763" ] }, { "annotation": null, "id": "r:789", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:768", "s:769" ], "params": {}, "parent": null, "results": [ "s:765" ] }, { "annotation": null, "id": "r:790", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:766" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:789", "results": [ "s:765" ] }, { "annotation": null, "id": "r:791", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:766" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:790", "results": [ "s:765" ] }, { "annotation": null, "id": "r:792", "op": "linear.sum_nodes", "operands": [ "s:768", "s:769" ], "params": { "max_fanout": null }, "parent": "r:789", "results": [ "s:766" ] }, { "annotation": null, "id": "r:793", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 9.5, "true_level": 1.0 }, "parent": null, "results": [ "s:768" ] }, { "annotation": null, "id": "r:794", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:793", "results": [ "s:768" ] }, { "annotation": null, "id": "r:795", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:770" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -10.5, "true_level": 1.0 }, "parent": null, "results": [ "s:769" ] }, { "annotation": null, "id": "r:796", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:770" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:795", "results": [ "s:769" ] }, { "annotation": null, "id": "r:797", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:770" ] }, { "annotation": null, "id": "r:798", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:775", "s:794", "s:813" ], "params": {}, "parent": null, "results": [ "s:771" ] }, { "annotation": null, "id": "r:799", "op": "linear.sum_nodes", "operands": [ "s:773", "s:792", "s:811" ], "params": { "max_fanout": null }, "parent": "r:798", "results": [ "s:771" ] }, { "annotation": null, "id": "r:800", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:775" ], "params": {}, "parent": "r:798", "results": [ "s:773" ] }, { "annotation": null, "id": "r:801", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:774" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:800", "results": [ "s:773" ] }, { "annotation": null, "id": "r:802", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:776" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:775" ] }, { "annotation": null, "id": "r:803", "op": "linear.bool_to_01", "operands": [ "s:778" ], "params": {}, "parent": null, "results": [ "s:776" ] }, { "annotation": null, "id": "r:804", "op": "linear.multiply_const", "operands": [ "s:777" ], "params": { "scalar": 0.5 }, "parent": "r:803", "results": [ "s:776" ] }, { "annotation": null, "id": "r:805", "op": "linear.add_const", "operands": [ "s:778" ], "params": { "scalar": 1.0 }, "parent": "r:803", "results": [ "s:777" ] }, { "annotation": null, "id": "r:806", "op": "swiglu.map_select.in_range", "operands": [ "s:780", "s:791" ], "params": { "n_slots": 7 }, "parent": null, "results": [ "s:778" ] }, { "annotation": null, "id": "r:807", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:779" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 28 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 7 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 7 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:806", "results": [ "s:778" ] }, { "annotation": null, "id": "r:808", "op": "linear.add", "operands": [ "s:781", "s:782" ], "params": {}, "parent": null, "results": [ "s:780" ] }, { "annotation": null, "id": "r:809", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:781" ] }, { "annotation": null, "id": "r:810", "op": "linear.bool_to_01", "operands": [ "s:784" ], "params": {}, "parent": null, "results": [ "s:782" ] }, { "annotation": null, "id": "r:811", "op": "linear.multiply_const", "operands": [ "s:783" ], "params": { "scalar": 0.5 }, "parent": "r:810", "results": [ "s:782" ] }, { "annotation": null, "id": "r:812", "op": "linear.add_const", "operands": [ "s:784" ], "params": { "scalar": 1.0 }, "parent": "r:810", "results": [ "s:783" ] }, { "annotation": null, "id": "r:813", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:787", "s:788" ], "params": {}, "parent": null, "results": [ "s:784" ] }, { "annotation": null, "id": "r:814", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:785" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:813", "results": [ "s:784" ] }, { "annotation": null, "id": "r:815", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:785" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:814", "results": [ "s:784" ] }, { "annotation": null, "id": "r:816", "op": "linear.sum_nodes", "operands": [ "s:787", "s:788" ], "params": { "max_fanout": null }, "parent": "r:813", "results": [ "s:785" ] }, { "annotation": null, "id": "r:817", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:781" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:787" ] }, { "annotation": null, "id": "r:818", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:781" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:817", "results": [ "s:787" ] }, { "annotation": null, "id": "r:819", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:789" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:788" ] }, { "annotation": null, "id": "r:820", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:789" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:819", "results": [ "s:788" ] }, { "annotation": null, "id": "r:821", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -3, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:789" ] }, { "annotation": null, "id": "r:822", "op": "linear.add_const", "operands": [ "s:780" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:791" ] }, { "annotation": null, "id": "r:823", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:794" ], "params": {}, "parent": "r:798", "results": [ "s:792" ] }, { "annotation": null, "id": "r:824", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:793" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:823", "results": [ "s:792" ] }, { "annotation": null, "id": "r:825", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:795" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:794" ] }, { "annotation": null, "id": "r:826", "op": "linear.bool_to_01", "operands": [ "s:797" ], "params": {}, "parent": null, "results": [ "s:795" ] }, { "annotation": null, "id": "r:827", "op": "linear.multiply_const", "operands": [ "s:796" ], "params": { "scalar": 0.5 }, "parent": "r:826", "results": [ "s:795" ] }, { "annotation": null, "id": "r:828", "op": "linear.add_const", "operands": [ "s:797" ], "params": { "scalar": 1.0 }, "parent": "r:826", "results": [ "s:796" ] }, { "annotation": null, "id": "r:829", "op": "swiglu.map_select.in_range", "operands": [ "s:799", "s:810" ], "params": { "n_slots": 7 }, "parent": null, "results": [ "s:797" ] }, { "annotation": null, "id": "r:830", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:798" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 28 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 7 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 7 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:829", "results": [ "s:797" ] }, { "annotation": null, "id": "r:831", "op": "linear.add", "operands": [ "s:800", "s:801" ], "params": {}, "parent": null, "results": [ "s:799" ] }, { "annotation": null, "id": "r:832", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:800" ] }, { "annotation": null, "id": "r:833", "op": "linear.bool_to_01", "operands": [ "s:803" ], "params": {}, "parent": null, "results": [ "s:801" ] }, { "annotation": null, "id": "r:834", "op": "linear.multiply_const", "operands": [ "s:802" ], "params": { "scalar": 0.5 }, "parent": "r:833", "results": [ "s:801" ] }, { "annotation": null, "id": "r:835", "op": "linear.add_const", "operands": [ "s:803" ], "params": { "scalar": 1.0 }, "parent": "r:833", "results": [ "s:802" ] }, { "annotation": null, "id": "r:836", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:806", "s:807" ], "params": {}, "parent": null, "results": [ "s:803" ] }, { "annotation": null, "id": "r:837", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:804" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:836", "results": [ "s:803" ] }, { "annotation": null, "id": "r:838", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:804" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:837", "results": [ "s:803" ] }, { "annotation": null, "id": "r:839", "op": "linear.sum_nodes", "operands": [ "s:806", "s:807" ], "params": { "max_fanout": null }, "parent": "r:836", "results": [ "s:804" ] }, { "annotation": null, "id": "r:840", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:800" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:806" ] }, { "annotation": null, "id": "r:841", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:800" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:840", "results": [ "s:806" ] }, { "annotation": null, "id": "r:842", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:808" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:807" ] }, { "annotation": null, "id": "r:843", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:808" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:842", "results": [ "s:807" ] }, { "annotation": null, "id": "r:844", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -2, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:808" ] }, { "annotation": null, "id": "r:845", "op": "linear.add_const", "operands": [ "s:799" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:810" ] }, { "annotation": null, "id": "r:846", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:813" ], "params": {}, "parent": "r:798", "results": [ "s:811" ] }, { "annotation": null, "id": "r:847", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:812" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:846", "results": [ "s:811" ] }, { "annotation": null, "id": "r:848", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:814" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:813" ] }, { "annotation": null, "id": "r:849", "op": "linear.bool_to_01", "operands": [ "s:816" ], "params": {}, "parent": null, "results": [ "s:814" ] }, { "annotation": null, "id": "r:850", "op": "linear.multiply_const", "operands": [ "s:815" ], "params": { "scalar": 0.5 }, "parent": "r:849", "results": [ "s:814" ] }, { "annotation": null, "id": "r:851", "op": "linear.add_const", "operands": [ "s:816" ], "params": { "scalar": 1.0 }, "parent": "r:849", "results": [ "s:815" ] }, { "annotation": null, "id": "r:852", "op": "swiglu.map_select.in_range", "operands": [ "s:818", "s:822" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:816" ] }, { "annotation": null, "id": "r:853", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:817" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:852", "results": [ "s:816" ] }, { "annotation": null, "id": "r:854", "op": "linear.add", "operands": [ "s:391", "s:819" ], "params": {}, "parent": null, "results": [ "s:818" ] }, { "annotation": null, "id": "r:855", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:820" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:819" ] }, { "annotation": null, "id": "r:856", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -7, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:820" ] }, { "annotation": null, "id": "r:857", "op": "linear.add_const", "operands": [ "s:818" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:822" ] }, { "annotation": null, "id": "r:858", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:825", "s:831" ], "params": {}, "parent": null, "results": [ "s:823" ] }, { "annotation": null, "id": "r:859", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:824" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:858", "results": [ "s:823" ] }, { "annotation": null, "id": "r:860", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:828", "s:829" ], "params": {}, "parent": null, "results": [ "s:825" ] }, { "annotation": null, "id": "r:861", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:826" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:860", "results": [ "s:825" ] }, { "annotation": null, "id": "r:862", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:826" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:861", "results": [ "s:825" ] }, { "annotation": null, "id": "r:863", "op": "linear.sum_nodes", "operands": [ "s:828", "s:829" ], "params": { "max_fanout": null }, "parent": "r:860", "results": [ "s:826" ] }, { "annotation": null, "id": "r:864", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 10.5, "true_level": 1.0 }, "parent": null, "results": [ "s:828" ] }, { "annotation": null, "id": "r:865", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:864", "results": [ "s:828" ] }, { "annotation": null, "id": "r:866", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:830" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -11.5, "true_level": 1.0 }, "parent": null, "results": [ "s:829" ] }, { "annotation": null, "id": "r:867", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:830" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:866", "results": [ "s:829" ] }, { "annotation": null, "id": "r:868", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:830" ] }, { "annotation": null, "id": "r:869", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:835", "s:854", "s:873" ], "params": {}, "parent": null, "results": [ "s:831" ] }, { "annotation": null, "id": "r:870", "op": "linear.sum_nodes", "operands": [ "s:833", "s:852", "s:871" ], "params": { "max_fanout": null }, "parent": "r:869", "results": [ "s:831" ] }, { "annotation": null, "id": "r:871", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:835" ], "params": {}, "parent": "r:869", "results": [ "s:833" ] }, { "annotation": null, "id": "r:872", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:834" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:871", "results": [ "s:833" ] }, { "annotation": null, "id": "r:873", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:836" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:835" ] }, { "annotation": null, "id": "r:874", "op": "linear.bool_to_01", "operands": [ "s:838" ], "params": {}, "parent": null, "results": [ "s:836" ] }, { "annotation": null, "id": "r:875", "op": "linear.multiply_const", "operands": [ "s:837" ], "params": { "scalar": 0.5 }, "parent": "r:874", "results": [ "s:836" ] }, { "annotation": null, "id": "r:876", "op": "linear.add_const", "operands": [ "s:838" ], "params": { "scalar": 1.0 }, "parent": "r:874", "results": [ "s:837" ] }, { "annotation": null, "id": "r:877", "op": "swiglu.map_select.in_range", "operands": [ "s:840", "s:851" ], "params": { "n_slots": 7 }, "parent": null, "results": [ "s:838" ] }, { "annotation": null, "id": "r:878", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:839" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 28 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 7 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 7 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:877", "results": [ "s:838" ] }, { "annotation": null, "id": "r:879", "op": "linear.add", "operands": [ "s:841", "s:842" ], "params": {}, "parent": null, "results": [ "s:840" ] }, { "annotation": null, "id": "r:880", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:841" ] }, { "annotation": null, "id": "r:881", "op": "linear.bool_to_01", "operands": [ "s:844" ], "params": {}, "parent": null, "results": [ "s:842" ] }, { "annotation": null, "id": "r:882", "op": "linear.multiply_const", "operands": [ "s:843" ], "params": { "scalar": 0.5 }, "parent": "r:881", "results": [ "s:842" ] }, { "annotation": null, "id": "r:883", "op": "linear.add_const", "operands": [ "s:844" ], "params": { "scalar": 1.0 }, "parent": "r:881", "results": [ "s:843" ] }, { "annotation": null, "id": "r:884", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:847", "s:848" ], "params": {}, "parent": null, "results": [ "s:844" ] }, { "annotation": null, "id": "r:885", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:845" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:884", "results": [ "s:844" ] }, { "annotation": null, "id": "r:886", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:845" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:885", "results": [ "s:844" ] }, { "annotation": null, "id": "r:887", "op": "linear.sum_nodes", "operands": [ "s:847", "s:848" ], "params": { "max_fanout": null }, "parent": "r:884", "results": [ "s:845" ] }, { "annotation": null, "id": "r:888", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:841" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.5, "true_level": 1.0 }, "parent": null, "results": [ "s:847" ] }, { "annotation": null, "id": "r:889", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:841" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:888", "results": [ "s:847" ] }, { "annotation": null, "id": "r:890", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:849" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:848" ] }, { "annotation": null, "id": "r:891", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:849" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:890", "results": [ "s:848" ] }, { "annotation": null, "id": "r:892", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -3, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:849" ] }, { "annotation": null, "id": "r:893", "op": "linear.add_const", "operands": [ "s:840" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:851" ] }, { "annotation": null, "id": "r:894", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:854" ], "params": {}, "parent": "r:869", "results": [ "s:852" ] }, { "annotation": null, "id": "r:895", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:853" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:894", "results": [ "s:852" ] }, { "annotation": null, "id": "r:896", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:855" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:854" ] }, { "annotation": null, "id": "r:897", "op": "linear.bool_to_01", "operands": [ "s:857" ], "params": {}, "parent": null, "results": [ "s:855" ] }, { "annotation": null, "id": "r:898", "op": "linear.multiply_const", "operands": [ "s:856" ], "params": { "scalar": 0.5 }, "parent": "r:897", "results": [ "s:855" ] }, { "annotation": null, "id": "r:899", "op": "linear.add_const", "operands": [ "s:857" ], "params": { "scalar": 1.0 }, "parent": "r:897", "results": [ "s:856" ] }, { "annotation": null, "id": "r:900", "op": "swiglu.map_select.in_range", "operands": [ "s:859", "s:870" ], "params": { "n_slots": 7 }, "parent": null, "results": [ "s:857" ] }, { "annotation": null, "id": "r:901", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:858" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 28 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 7 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 7 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:900", "results": [ "s:857" ] }, { "annotation": null, "id": "r:902", "op": "linear.add", "operands": [ "s:860", "s:861" ], "params": {}, "parent": null, "results": [ "s:859" ] }, { "annotation": null, "id": "r:903", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:860" ] }, { "annotation": null, "id": "r:904", "op": "linear.bool_to_01", "operands": [ "s:863" ], "params": {}, "parent": null, "results": [ "s:861" ] }, { "annotation": null, "id": "r:905", "op": "linear.multiply_const", "operands": [ "s:862" ], "params": { "scalar": 0.5 }, "parent": "r:904", "results": [ "s:861" ] }, { "annotation": null, "id": "r:906", "op": "linear.add_const", "operands": [ "s:863" ], "params": { "scalar": 1.0 }, "parent": "r:904", "results": [ "s:862" ] }, { "annotation": null, "id": "r:907", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:866", "s:867" ], "params": {}, "parent": null, "results": [ "s:863" ] }, { "annotation": null, "id": "r:908", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:864" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:907", "results": [ "s:863" ] }, { "annotation": null, "id": "r:909", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:864" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:908", "results": [ "s:863" ] }, { "annotation": null, "id": "r:910", "op": "linear.sum_nodes", "operands": [ "s:866", "s:867" ], "params": { "max_fanout": null }, "parent": "r:907", "results": [ "s:864" ] }, { "annotation": null, "id": "r:911", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:860" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:866" ] }, { "annotation": null, "id": "r:912", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:860" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:911", "results": [ "s:866" ] }, { "annotation": null, "id": "r:913", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:868" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:867" ] }, { "annotation": null, "id": "r:914", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:868" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:913", "results": [ "s:867" ] }, { "annotation": null, "id": "r:915", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -2, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:868" ] }, { "annotation": null, "id": "r:916", "op": "linear.add_const", "operands": [ "s:859" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:870" ] }, { "annotation": null, "id": "r:917", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:873" ], "params": {}, "parent": "r:869", "results": [ "s:871" ] }, { "annotation": null, "id": "r:918", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:872" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:917", "results": [ "s:871" ] }, { "annotation": null, "id": "r:919", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:874" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:873" ] }, { "annotation": null, "id": "r:920", "op": "linear.bool_to_01", "operands": [ "s:876" ], "params": {}, "parent": null, "results": [ "s:874" ] }, { "annotation": null, "id": "r:921", "op": "linear.multiply_const", "operands": [ "s:875" ], "params": { "scalar": 0.5 }, "parent": "r:920", "results": [ "s:874" ] }, { "annotation": null, "id": "r:922", "op": "linear.add_const", "operands": [ "s:876" ], "params": { "scalar": 1.0 }, "parent": "r:920", "results": [ "s:875" ] }, { "annotation": null, "id": "r:923", "op": "swiglu.map_select.in_range", "operands": [ "s:878", "s:882" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:876" ] }, { "annotation": null, "id": "r:924", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:877" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:923", "results": [ "s:876" ] }, { "annotation": null, "id": "r:925", "op": "linear.add", "operands": [ "s:345", "s:879" ], "params": {}, "parent": null, "results": [ "s:878" ] }, { "annotation": null, "id": "r:926", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:880" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:879" ] }, { "annotation": null, "id": "r:927", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -9, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:880" ] }, { "annotation": null, "id": "r:928", "op": "linear.add_const", "operands": [ "s:878" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:882" ] }, { "annotation": null, "id": "r:929", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:885", "s:891" ], "params": {}, "parent": null, "results": [ "s:883" ] }, { "annotation": null, "id": "r:930", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:884" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:929", "results": [ "s:883" ] }, { "annotation": null, "id": "r:931", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:888", "s:889" ], "params": {}, "parent": null, "results": [ "s:885" ] }, { "annotation": null, "id": "r:932", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:886" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:931", "results": [ "s:885" ] }, { "annotation": null, "id": "r:933", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:886" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:932", "results": [ "s:885" ] }, { "annotation": null, "id": "r:934", "op": "linear.sum_nodes", "operands": [ "s:888", "s:889" ], "params": { "max_fanout": null }, "parent": "r:931", "results": [ "s:886" ] }, { "annotation": null, "id": "r:935", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 11.5, "true_level": 1.0 }, "parent": null, "results": [ "s:888" ] }, { "annotation": null, "id": "r:936", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:935", "results": [ "s:888" ] }, { "annotation": null, "id": "r:937", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:890" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -12.5, "true_level": 1.0 }, "parent": null, "results": [ "s:889" ] }, { "annotation": null, "id": "r:938", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:890" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:937", "results": [ "s:889" ] }, { "annotation": null, "id": "r:939", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:890" ] }, { "annotation": null, "id": "r:940", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:895", "s:914", "s:933" ], "params": {}, "parent": null, "results": [ "s:891" ] }, { "annotation": null, "id": "r:941", "op": "linear.sum_nodes", "operands": [ "s:893", "s:912", "s:931" ], "params": { "max_fanout": null }, "parent": "r:940", "results": [ "s:891" ] }, { "annotation": null, "id": "r:942", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:895" ], "params": {}, "parent": "r:940", "results": [ "s:893" ] }, { "annotation": null, "id": "r:943", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:894" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:942", "results": [ "s:893" ] }, { "annotation": null, "id": "r:944", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:896" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:895" ] }, { "annotation": null, "id": "r:945", "op": "linear.bool_to_01", "operands": [ "s:898" ], "params": {}, "parent": null, "results": [ "s:896" ] }, { "annotation": null, "id": "r:946", "op": "linear.multiply_const", "operands": [ "s:897" ], "params": { "scalar": 0.5 }, "parent": "r:945", "results": [ "s:896" ] }, { "annotation": null, "id": "r:947", "op": "linear.add_const", "operands": [ "s:898" ], "params": { "scalar": 1.0 }, "parent": "r:945", "results": [ "s:897" ] }, { "annotation": null, "id": "r:948", "op": "swiglu.map_select.in_range", "operands": [ "s:900", "s:911" ], "params": { "n_slots": 7 }, "parent": null, "results": [ "s:898" ] }, { "annotation": null, "id": "r:949", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:899" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 28 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 7 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 7 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:948", "results": [ "s:898" ] }, { "annotation": null, "id": "r:950", "op": "linear.add", "operands": [ "s:901", "s:902" ], "params": {}, "parent": null, "results": [ "s:900" ] }, { "annotation": null, "id": "r:951", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:901" ] }, { "annotation": null, "id": "r:952", "op": "linear.bool_to_01", "operands": [ "s:904" ], "params": {}, "parent": null, "results": [ "s:902" ] }, { "annotation": null, "id": "r:953", "op": "linear.multiply_const", "operands": [ "s:903" ], "params": { "scalar": 0.5 }, "parent": "r:952", "results": [ "s:902" ] }, { "annotation": null, "id": "r:954", "op": "linear.add_const", "operands": [ "s:904" ], "params": { "scalar": 1.0 }, "parent": "r:952", "results": [ "s:903" ] }, { "annotation": null, "id": "r:955", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:907", "s:908" ], "params": {}, "parent": null, "results": [ "s:904" ] }, { "annotation": null, "id": "r:956", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:905" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:955", "results": [ "s:904" ] }, { "annotation": null, "id": "r:957", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:905" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:956", "results": [ "s:904" ] }, { "annotation": null, "id": "r:958", "op": "linear.sum_nodes", "operands": [ "s:907", "s:908" ], "params": { "max_fanout": null }, "parent": "r:955", "results": [ "s:905" ] }, { "annotation": null, "id": "r:959", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:901" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 2.5, "true_level": 1.0 }, "parent": null, "results": [ "s:907" ] }, { "annotation": null, "id": "r:960", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:901" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:959", "results": [ "s:907" ] }, { "annotation": null, "id": "r:961", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:909" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:908" ] }, { "annotation": null, "id": "r:962", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:909" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:961", "results": [ "s:908" ] }, { "annotation": null, "id": "r:963", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -3, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:909" ] }, { "annotation": null, "id": "r:964", "op": "linear.add_const", "operands": [ "s:900" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:911" ] }, { "annotation": null, "id": "r:965", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:914" ], "params": {}, "parent": "r:940", "results": [ "s:912" ] }, { "annotation": null, "id": "r:966", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:913" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:965", "results": [ "s:912" ] }, { "annotation": null, "id": "r:967", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:915" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:914" ] }, { "annotation": null, "id": "r:968", "op": "linear.bool_to_01", "operands": [ "s:917" ], "params": {}, "parent": null, "results": [ "s:915" ] }, { "annotation": null, "id": "r:969", "op": "linear.multiply_const", "operands": [ "s:916" ], "params": { "scalar": 0.5 }, "parent": "r:968", "results": [ "s:915" ] }, { "annotation": null, "id": "r:970", "op": "linear.add_const", "operands": [ "s:917" ], "params": { "scalar": 1.0 }, "parent": "r:968", "results": [ "s:916" ] }, { "annotation": null, "id": "r:971", "op": "swiglu.map_select.in_range", "operands": [ "s:919", "s:930" ], "params": { "n_slots": 7 }, "parent": null, "results": [ "s:917" ] }, { "annotation": null, "id": "r:972", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:918" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 28 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 7 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 7 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:971", "results": [ "s:917" ] }, { "annotation": null, "id": "r:973", "op": "linear.add", "operands": [ "s:920", "s:921" ], "params": {}, "parent": null, "results": [ "s:919" ] }, { "annotation": null, "id": "r:974", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:920" ] }, { "annotation": null, "id": "r:975", "op": "linear.bool_to_01", "operands": [ "s:923" ], "params": {}, "parent": null, "results": [ "s:921" ] }, { "annotation": null, "id": "r:976", "op": "linear.multiply_const", "operands": [ "s:922" ], "params": { "scalar": 0.5 }, "parent": "r:975", "results": [ "s:921" ] }, { "annotation": null, "id": "r:977", "op": "linear.add_const", "operands": [ "s:923" ], "params": { "scalar": 1.0 }, "parent": "r:975", "results": [ "s:922" ] }, { "annotation": null, "id": "r:978", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:926", "s:927" ], "params": {}, "parent": null, "results": [ "s:923" ] }, { "annotation": null, "id": "r:979", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:924" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:978", "results": [ "s:923" ] }, { "annotation": null, "id": "r:980", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:924" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:979", "results": [ "s:923" ] }, { "annotation": null, "id": "r:981", "op": "linear.sum_nodes", "operands": [ "s:926", "s:927" ], "params": { "max_fanout": null }, "parent": "r:978", "results": [ "s:924" ] }, { "annotation": null, "id": "r:982", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:920" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.5, "true_level": 1.0 }, "parent": null, "results": [ "s:926" ] }, { "annotation": null, "id": "r:983", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:920" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:982", "results": [ "s:926" ] }, { "annotation": null, "id": "r:984", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:928" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:927" ] }, { "annotation": null, "id": "r:985", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:928" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:984", "results": [ "s:927" ] }, { "annotation": null, "id": "r:986", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -2, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:928" ] }, { "annotation": null, "id": "r:987", "op": "linear.add_const", "operands": [ "s:919" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:930" ] }, { "annotation": null, "id": "r:988", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:933" ], "params": {}, "parent": "r:940", "results": [ "s:931" ] }, { "annotation": null, "id": "r:989", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:932" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:988", "results": [ "s:931" ] }, { "annotation": null, "id": "r:990", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:934" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:933" ] }, { "annotation": null, "id": "r:991", "op": "linear.bool_to_01", "operands": [ "s:936" ], "params": {}, "parent": null, "results": [ "s:934" ] }, { "annotation": null, "id": "r:992", "op": "linear.multiply_const", "operands": [ "s:935" ], "params": { "scalar": 0.5 }, "parent": "r:991", "results": [ "s:934" ] }, { "annotation": null, "id": "r:993", "op": "linear.add_const", "operands": [ "s:936" ], "params": { "scalar": 1.0 }, "parent": "r:991", "results": [ "s:935" ] }, { "annotation": null, "id": "r:994", "op": "swiglu.map_select.in_range", "operands": [ "s:938", "s:940" ], "params": { "n_slots": 61 }, "parent": null, "results": [ "s:936" ] }, { "annotation": null, "id": "r:995", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:937" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 244 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 61 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 244, 61 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:994", "results": [ "s:936" ] }, { "annotation": null, "id": "r:996", "op": "linear.add", "operands": [ "s:270", "s:939" ], "params": {}, "parent": null, "results": [ "s:938" ] }, { "annotation": null, "id": "r:997", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:939" ] }, { "annotation": null, "id": "r:998", "op": "linear.add_const", "operands": [ "s:938" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:940" ] }, { "annotation": null, "id": "r:999", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:943", "s:949" ], "params": {}, "parent": null, "results": [ "s:941" ] }, { "annotation": null, "id": "r:1000", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:942" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:999", "results": [ "s:941" ] }, { "annotation": null, "id": "r:1001", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:946", "s:947" ], "params": {}, "parent": null, "results": [ "s:943" ] }, { "annotation": null, "id": "r:1002", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:944" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1001", "results": [ "s:943" ] }, { "annotation": null, "id": "r:1003", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:944" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1002", "results": [ "s:943" ] }, { "annotation": null, "id": "r:1004", "op": "linear.sum_nodes", "operands": [ "s:946", "s:947" ], "params": { "max_fanout": null }, "parent": "r:1001", "results": [ "s:944" ] }, { "annotation": null, "id": "r:1005", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 12.5, "true_level": 1.0 }, "parent": null, "results": [ "s:946" ] }, { "annotation": null, "id": "r:1006", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1005", "results": [ "s:946" ] }, { "annotation": null, "id": "r:1007", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:948" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -13.5, "true_level": 1.0 }, "parent": null, "results": [ "s:947" ] }, { "annotation": null, "id": "r:1008", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:948" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1007", "results": [ "s:947" ] }, { "annotation": null, "id": "r:1009", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:948" ] }, { "annotation": null, "id": "r:1010", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:953", "s:956", "s:959" ], "params": {}, "parent": null, "results": [ "s:949" ] }, { "annotation": null, "id": "r:1011", "op": "linear.sum_nodes", "operands": [ "s:951", "s:954", "s:957" ], "params": { "max_fanout": null }, "parent": "r:1010", "results": [ "s:949" ] }, { "annotation": null, "id": "r:1012", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:953" ], "params": {}, "parent": "r:1010", "results": [ "s:951" ] }, { "annotation": null, "id": "r:1013", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:952" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1012", "results": [ "s:951" ] }, { "annotation": null, "id": "r:1014", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:953" ] }, { "annotation": null, "id": "r:1015", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:956" ], "params": {}, "parent": "r:1010", "results": [ "s:954" ] }, { "annotation": null, "id": "r:1016", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:955" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1015", "results": [ "s:954" ] }, { "annotation": null, "id": "r:1017", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:956" ] }, { "annotation": null, "id": "r:1018", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:959" ], "params": {}, "parent": "r:1010", "results": [ "s:957" ] }, { "annotation": null, "id": "r:1019", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:958" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1018", "results": [ "s:957" ] }, { "annotation": null, "id": "r:1020", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:960" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:959" ] }, { "annotation": null, "id": "r:1021", "op": "linear.bool_to_01", "operands": [ "s:962" ], "params": {}, "parent": null, "results": [ "s:960" ] }, { "annotation": null, "id": "r:1022", "op": "linear.multiply_const", "operands": [ "s:961" ], "params": { "scalar": 0.5 }, "parent": "r:1021", "results": [ "s:960" ] }, { "annotation": null, "id": "r:1023", "op": "linear.add_const", "operands": [ "s:962" ], "params": { "scalar": 1.0 }, "parent": "r:1021", "results": [ "s:961" ] }, { "annotation": null, "id": "r:1024", "op": "swiglu.map_select.in_range", "operands": [ "s:964", "s:975" ], "params": { "n_slots": 7 }, "parent": null, "results": [ "s:962" ] }, { "annotation": null, "id": "r:1025", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:963" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 28 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 7 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 7 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1024", "results": [ "s:962" ] }, { "annotation": null, "id": "r:1026", "op": "linear.add", "operands": [ "s:965", "s:966" ], "params": {}, "parent": null, "results": [ "s:964" ] }, { "annotation": null, "id": "r:1027", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:965" ] }, { "annotation": null, "id": "r:1028", "op": "linear.bool_to_01", "operands": [ "s:968" ], "params": {}, "parent": null, "results": [ "s:966" ] }, { "annotation": null, "id": "r:1029", "op": "linear.multiply_const", "operands": [ "s:967" ], "params": { "scalar": 0.5 }, "parent": "r:1028", "results": [ "s:966" ] }, { "annotation": null, "id": "r:1030", "op": "linear.add_const", "operands": [ "s:968" ], "params": { "scalar": 1.0 }, "parent": "r:1028", "results": [ "s:967" ] }, { "annotation": null, "id": "r:1031", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:971", "s:972" ], "params": {}, "parent": null, "results": [ "s:968" ] }, { "annotation": null, "id": "r:1032", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:969" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1031", "results": [ "s:968" ] }, { "annotation": null, "id": "r:1033", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:969" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1032", "results": [ "s:968" ] }, { "annotation": null, "id": "r:1034", "op": "linear.sum_nodes", "operands": [ "s:971", "s:972" ], "params": { "max_fanout": null }, "parent": "r:1031", "results": [ "s:969" ] }, { "annotation": null, "id": "r:1035", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:965" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:971" ] }, { "annotation": null, "id": "r:1036", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:965" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1035", "results": [ "s:971" ] }, { "annotation": null, "id": "r:1037", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:973" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:972" ] }, { "annotation": null, "id": "r:1038", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:973" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1037", "results": [ "s:972" ] }, { "annotation": null, "id": "r:1039", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -5, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:973" ] }, { "annotation": null, "id": "r:1040", "op": "linear.add_const", "operands": [ "s:964" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:975" ] }, { "annotation": null, "id": "r:1041", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:978", "s:984" ], "params": {}, "parent": null, "results": [ "s:976" ] }, { "annotation": null, "id": "r:1042", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:977" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1041", "results": [ "s:976" ] }, { "annotation": null, "id": "r:1043", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:981", "s:982" ], "params": {}, "parent": null, "results": [ "s:978" ] }, { "annotation": null, "id": "r:1044", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:979" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1043", "results": [ "s:978" ] }, { "annotation": null, "id": "r:1045", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:979" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1044", "results": [ "s:978" ] }, { "annotation": null, "id": "r:1046", "op": "linear.sum_nodes", "operands": [ "s:981", "s:982" ], "params": { "max_fanout": null }, "parent": "r:1043", "results": [ "s:979" ] }, { "annotation": null, "id": "r:1047", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 13.5, "true_level": 1.0 }, "parent": null, "results": [ "s:981" ] }, { "annotation": null, "id": "r:1048", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1047", "results": [ "s:981" ] }, { "annotation": null, "id": "r:1049", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:983" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -14.5, "true_level": 1.0 }, "parent": null, "results": [ "s:982" ] }, { "annotation": null, "id": "r:1050", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:983" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1049", "results": [ "s:982" ] }, { "annotation": null, "id": "r:1051", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:983" ] }, { "annotation": null, "id": "r:1052", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:988", "s:1013", "s:1046" ], "params": {}, "parent": null, "results": [ "s:984" ] }, { "annotation": null, "id": "r:1053", "op": "linear.sum_nodes", "operands": [ "s:986", "s:1011", "s:1044" ], "params": { "max_fanout": null }, "parent": "r:1052", "results": [ "s:984" ] }, { "annotation": null, "id": "r:1054", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:988" ], "params": {}, "parent": "r:1052", "results": [ "s:986" ] }, { "annotation": null, "id": "r:1055", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:987" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1054", "results": [ "s:986" ] }, { "annotation": null, "id": "r:1056", "op": "swiglu.map_select.select", "operands": [ "s:990", "s:998", "s:999" ], "params": {}, "parent": null, "results": [ "s:988" ] }, { "annotation": null, "id": "r:1057", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:989" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1056", "results": [ "s:988" ] }, { "annotation": null, "id": "r:1058", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:991" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 3.5, "true_level": 1.0 }, "parent": null, "results": [ "s:990" ] }, { "annotation": null, "id": "r:1059", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:991" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1058", "results": [ "s:990" ] }, { "annotation": null, "id": "r:1060", "op": "linear.add_const", "operands": [ "s:992" ], "params": { "scalar": 0.0 }, "parent": null, "results": [ "s:991" ] }, { "annotation": null, "id": "r:1061", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:994", "s:997" ], "params": {}, "parent": null, "results": [ "s:992" ] }, { "annotation": null, "id": "r:1062", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:993" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1061", "results": [ "s:992" ] }, { "annotation": null, "id": "r:1063", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:995" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:994" ] }, { "annotation": null, "id": "r:1064", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -1, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:995" ] }, { "annotation": null, "id": "r:1065", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:997" ] }, { "annotation": null, "id": "r:1066", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:998" ] }, { "annotation": null, "id": "r:1067", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1000", "s:1005", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:999" ] }, { "annotation": null, "id": "r:1068", "op": "linear.bool_to_01", "operands": [ "s:1002" ], "params": {}, "parent": null, "results": [ "s:1000" ] }, { "annotation": null, "id": "r:1069", "op": "linear.multiply_const", "operands": [ "s:1001" ], "params": { "scalar": 0.5 }, "parent": "r:1068", "results": [ "s:1000" ] }, { "annotation": null, "id": "r:1070", "op": "linear.add_const", "operands": [ "s:1002" ], "params": { "scalar": 1.0 }, "parent": "r:1068", "results": [ "s:1001" ] }, { "annotation": null, "id": "r:1071", "op": "swiglu.map_select.in_range", "operands": [ "s:991", "s:1004" ], "params": { "n_slots": 4 }, "parent": null, "results": [ "s:1002" ] }, { "annotation": null, "id": "r:1072", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1003" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 16 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 16, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 16, 4 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1071", "results": [ "s:1002" ] }, { "annotation": null, "id": "r:1073", "op": "linear.add_const", "operands": [ "s:991" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1004" ] }, { "annotation": null, "id": "r:1074", "op": "linear.bool_to_01", "operands": [ "s:1007" ], "params": {}, "parent": null, "results": [ "s:1005" ] }, { "annotation": null, "id": "r:1075", "op": "linear.multiply_const", "operands": [ "s:1006" ], "params": { "scalar": 0.5 }, "parent": "r:1074", "results": [ "s:1005" ] }, { "annotation": null, "id": "r:1076", "op": "linear.add_const", "operands": [ "s:1007" ], "params": { "scalar": 1.0 }, "parent": "r:1074", "results": [ "s:1006" ] }, { "annotation": null, "id": "r:1077", "op": "swiglu.map_select.in_range", "operands": [ "s:1009", "s:1010" ], "params": { "n_slots": 4 }, "parent": null, "results": [ "s:1007" ] }, { "annotation": null, "id": "r:1078", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1008" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 16 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 16, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 16, 4 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1077", "results": [ "s:1007" ] }, { "annotation": null, "id": "r:1079", "op": "linear.add_const", "operands": [ "s:18" ], "params": { "scalar": -6.0 }, "parent": null, "results": [ "s:1009" ] }, { "annotation": null, "id": "r:1080", "op": "linear.add_const", "operands": [ "s:1009" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1010" ] }, { "annotation": null, "id": "r:1081", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:1013" ], "params": {}, "parent": "r:1052", "results": [ "s:1011" ] }, { "annotation": null, "id": "r:1082", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1012" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1081", "results": [ "s:1011" ] }, { "annotation": null, "id": "r:1083", "op": "swiglu.map_select.select", "operands": [ "s:1015", "s:1019", "s:1043" ], "params": {}, "parent": null, "results": [ "s:1013" ] }, { "annotation": null, "id": "r:1084", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1014" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1083", "results": [ "s:1013" ] }, { "annotation": null, "id": "r:1085", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1016" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": null, "results": [ "s:1015" ] }, { "annotation": null, "id": "r:1086", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1016" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1085", "results": [ "s:1015" ] }, { "annotation": null, "id": "r:1087", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1017" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1016" ] }, { "annotation": null, "id": "r:1088", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -10, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1017" ] }, { "annotation": null, "id": "r:1089", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1020", "s:1037", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1019" ] }, { "annotation": null, "id": "r:1090", "op": "linear.bool_to_01", "operands": [ "s:1022" ], "params": {}, "parent": null, "results": [ "s:1020" ] }, { "annotation": null, "id": "r:1091", "op": "linear.multiply_const", "operands": [ "s:1021" ], "params": { "scalar": 0.5 }, "parent": "r:1090", "results": [ "s:1020" ] }, { "annotation": null, "id": "r:1092", "op": "linear.add_const", "operands": [ "s:1022" ], "params": { "scalar": 1.0 }, "parent": "r:1090", "results": [ "s:1021" ] }, { "annotation": null, "id": "r:1093", "op": "swiglu.map_select.in_range", "operands": [ "s:1024", "s:1036" ], "params": { "n_slots": 3 }, "parent": null, "results": [ "s:1022" ] }, { "annotation": null, "id": "r:1094", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1023" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1093", "results": [ "s:1022" ] }, { "annotation": null, "id": "r:1095", "op": "linear.subtract", "operands": [ "s:1025", "s:1033" ], "params": {}, "parent": null, "results": [ "s:1024" ] }, { "annotation": null, "id": "r:1096", "op": "linear.add", "operands": [ "s:1025", "s:1032" ], "params": {}, "parent": "r:1095", "results": [ "s:1024" ] }, { "annotation": null, "id": "r:1097", "op": "linear.add_const", "operands": [ "s:1026" ], "params": { "scalar": 0.0 }, "parent": null, "results": [ "s:1025" ] }, { "annotation": null, "id": "r:1098", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:1028", "s:1031" ], "params": {}, "parent": null, "results": [ "s:1026" ] }, { "annotation": null, "id": "r:1099", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1027" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1098", "results": [ "s:1026" ] }, { "annotation": null, "id": "r:1100", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1029" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1028" ] }, { "annotation": null, "id": "r:1101", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -1, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1029" ] }, { "annotation": null, "id": "r:1102", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:1031" ] }, { "annotation": null, "id": "r:1103", "op": "linear.negate", "operands": [ "s:1033" ], "params": {}, "parent": "r:1095", "results": [ "s:1032" ] }, { "annotation": null, "id": "r:1104", "op": "linear.bool_to_01", "operands": [ "s:1035" ], "params": {}, "parent": null, "results": [ "s:1033" ] }, { "annotation": null, "id": "r:1105", "op": "linear.multiply_const", "operands": [ "s:1034" ], "params": { "scalar": 0.5 }, "parent": "r:1104", "results": [ "s:1033" ] }, { "annotation": null, "id": "r:1106", "op": "linear.add_const", "operands": [ "s:1035" ], "params": { "scalar": 1.0 }, "parent": "r:1104", "results": [ "s:1034" ] }, { "annotation": null, "id": "r:1107", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:1015" ], "params": {}, "parent": null, "results": [ "s:1035" ] }, { "annotation": null, "id": "r:1108", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1015" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:1107", "results": [ "s:1035" ] }, { "annotation": null, "id": "r:1109", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1015" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1108", "results": [ "s:1035" ] }, { "annotation": null, "id": "r:1110", "op": "linear.add_const", "operands": [ "s:1024" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1036" ] }, { "annotation": null, "id": "r:1111", "op": "linear.bool_to_01", "operands": [ "s:1039" ], "params": {}, "parent": null, "results": [ "s:1037" ] }, { "annotation": null, "id": "r:1112", "op": "linear.multiply_const", "operands": [ "s:1038" ], "params": { "scalar": 0.5 }, "parent": "r:1111", "results": [ "s:1037" ] }, { "annotation": null, "id": "r:1113", "op": "linear.add_const", "operands": [ "s:1039" ], "params": { "scalar": 1.0 }, "parent": "r:1111", "results": [ "s:1038" ] }, { "annotation": null, "id": "r:1114", "op": "swiglu.map_select.in_range", "operands": [ "s:1041", "s:1042" ], "params": { "n_slots": 3 }, "parent": null, "results": [ "s:1039" ] }, { "annotation": null, "id": "r:1115", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1040" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1114", "results": [ "s:1039" ] }, { "annotation": null, "id": "r:1116", "op": "linear.add_const", "operands": [ "s:18" ], "params": { "scalar": -8.0 }, "parent": null, "results": [ "s:1041" ] }, { "annotation": null, "id": "r:1117", "op": "linear.add_const", "operands": [ "s:1041" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1042" ] }, { "annotation": null, "id": "r:1118", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1043" ] }, { "annotation": null, "id": "r:1119", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:1046" ], "params": {}, "parent": "r:1052", "results": [ "s:1044" ] }, { "annotation": null, "id": "r:1120", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1045" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1119", "results": [ "s:1044" ] }, { "annotation": null, "id": "r:1121", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1047" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1046" ] }, { "annotation": null, "id": "r:1122", "op": "linear.bool_to_01", "operands": [ "s:1049" ], "params": {}, "parent": null, "results": [ "s:1047" ] }, { "annotation": null, "id": "r:1123", "op": "linear.multiply_const", "operands": [ "s:1048" ], "params": { "scalar": 0.5 }, "parent": "r:1122", "results": [ "s:1047" ] }, { "annotation": null, "id": "r:1124", "op": "linear.add_const", "operands": [ "s:1049" ], "params": { "scalar": 1.0 }, "parent": "r:1122", "results": [ "s:1048" ] }, { "annotation": null, "id": "r:1125", "op": "swiglu.map_select.in_range", "operands": [ "s:1051", "s:1062" ], "params": { "n_slots": 7 }, "parent": null, "results": [ "s:1049" ] }, { "annotation": null, "id": "r:1126", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1050" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 28 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 7 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 7 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1125", "results": [ "s:1049" ] }, { "annotation": null, "id": "r:1127", "op": "linear.add", "operands": [ "s:1052", "s:1053" ], "params": {}, "parent": null, "results": [ "s:1051" ] }, { "annotation": null, "id": "r:1128", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1052" ] }, { "annotation": null, "id": "r:1129", "op": "linear.bool_to_01", "operands": [ "s:1055" ], "params": {}, "parent": null, "results": [ "s:1053" ] }, { "annotation": null, "id": "r:1130", "op": "linear.multiply_const", "operands": [ "s:1054" ], "params": { "scalar": 0.5 }, "parent": "r:1129", "results": [ "s:1053" ] }, { "annotation": null, "id": "r:1131", "op": "linear.add_const", "operands": [ "s:1055" ], "params": { "scalar": 1.0 }, "parent": "r:1129", "results": [ "s:1054" ] }, { "annotation": null, "id": "r:1132", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1058", "s:1059" ], "params": {}, "parent": null, "results": [ "s:1055" ] }, { "annotation": null, "id": "r:1133", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1056" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1132", "results": [ "s:1055" ] }, { "annotation": null, "id": "r:1134", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1056" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1133", "results": [ "s:1055" ] }, { "annotation": null, "id": "r:1135", "op": "linear.sum_nodes", "operands": [ "s:1058", "s:1059" ], "params": { "max_fanout": null }, "parent": "r:1132", "results": [ "s:1056" ] }, { "annotation": null, "id": "r:1136", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1052" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1058" ] }, { "annotation": null, "id": "r:1137", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1052" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1136", "results": [ "s:1058" ] }, { "annotation": null, "id": "r:1138", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:1060" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1059" ] }, { "annotation": null, "id": "r:1139", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1060" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1138", "results": [ "s:1059" ] }, { "annotation": null, "id": "r:1140", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -5, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1060" ] }, { "annotation": null, "id": "r:1141", "op": "linear.add_const", "operands": [ "s:1051" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1062" ] }, { "annotation": null, "id": "r:1142", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:1065", "s:1071" ], "params": {}, "parent": null, "results": [ "s:1063" ] }, { "annotation": null, "id": "r:1143", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1064" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1142", "results": [ "s:1063" ] }, { "annotation": null, "id": "r:1144", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1068", "s:1069" ], "params": {}, "parent": null, "results": [ "s:1065" ] }, { "annotation": null, "id": "r:1145", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1066" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1144", "results": [ "s:1065" ] }, { "annotation": null, "id": "r:1146", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1066" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1145", "results": [ "s:1065" ] }, { "annotation": null, "id": "r:1147", "op": "linear.sum_nodes", "operands": [ "s:1068", "s:1069" ], "params": { "max_fanout": null }, "parent": "r:1144", "results": [ "s:1066" ] }, { "annotation": null, "id": "r:1148", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 14.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1068" ] }, { "annotation": null, "id": "r:1149", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1148", "results": [ "s:1068" ] }, { "annotation": null, "id": "r:1150", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1070" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -15.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1069" ] }, { "annotation": null, "id": "r:1151", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1070" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1150", "results": [ "s:1069" ] }, { "annotation": null, "id": "r:1152", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:1070" ] }, { "annotation": null, "id": "r:1153", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:1075", "s:1093", "s:1121" ], "params": {}, "parent": null, "results": [ "s:1071" ] }, { "annotation": null, "id": "r:1154", "op": "linear.sum_nodes", "operands": [ "s:1073", "s:1091", "s:1119" ], "params": { "max_fanout": null }, "parent": "r:1153", "results": [ "s:1071" ] }, { "annotation": null, "id": "r:1155", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:1075" ], "params": {}, "parent": "r:1153", "results": [ "s:1073" ] }, { "annotation": null, "id": "r:1156", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1074" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1155", "results": [ "s:1073" ] }, { "annotation": null, "id": "r:1157", "op": "swiglu.map_select.select", "operands": [ "s:1077", "s:998", "s:1085" ], "params": {}, "parent": null, "results": [ "s:1075" ] }, { "annotation": null, "id": "r:1158", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1076" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1157", "results": [ "s:1075" ] }, { "annotation": null, "id": "r:1159", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1078" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 3.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1077" ] }, { "annotation": null, "id": "r:1160", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1078" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1159", "results": [ "s:1077" ] }, { "annotation": null, "id": "r:1161", "op": "linear.add_const", "operands": [ "s:1079" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1078" ] }, { "annotation": null, "id": "r:1162", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:1081", "s:1084" ], "params": {}, "parent": null, "results": [ "s:1079" ] }, { "annotation": null, "id": "r:1163", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1080" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1162", "results": [ "s:1079" ] }, { "annotation": null, "id": "r:1164", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1082" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1081" ] }, { "annotation": null, "id": "r:1165", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -2, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1082" ] }, { "annotation": null, "id": "r:1166", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:1084" ] }, { "annotation": null, "id": "r:1167", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1086", "s:1005", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1085" ] }, { "annotation": null, "id": "r:1168", "op": "linear.bool_to_01", "operands": [ "s:1088" ], "params": {}, "parent": null, "results": [ "s:1086" ] }, { "annotation": null, "id": "r:1169", "op": "linear.multiply_const", "operands": [ "s:1087" ], "params": { "scalar": 0.5 }, "parent": "r:1168", "results": [ "s:1086" ] }, { "annotation": null, "id": "r:1170", "op": "linear.add_const", "operands": [ "s:1088" ], "params": { "scalar": 1.0 }, "parent": "r:1168", "results": [ "s:1087" ] }, { "annotation": null, "id": "r:1171", "op": "swiglu.map_select.in_range", "operands": [ "s:1078", "s:1090" ], "params": { "n_slots": 4 }, "parent": null, "results": [ "s:1088" ] }, { "annotation": null, "id": "r:1172", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1089" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 16 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 16, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 16, 4 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1171", "results": [ "s:1088" ] }, { "annotation": null, "id": "r:1173", "op": "linear.add_const", "operands": [ "s:1078" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1090" ] }, { "annotation": null, "id": "r:1174", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:1093" ], "params": {}, "parent": "r:1153", "results": [ "s:1091" ] }, { "annotation": null, "id": "r:1175", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1092" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1174", "results": [ "s:1091" ] }, { "annotation": null, "id": "r:1176", "op": "swiglu.map_select.select", "operands": [ "s:1095", "s:1112", "s:1113" ], "params": {}, "parent": null, "results": [ "s:1093" ] }, { "annotation": null, "id": "r:1177", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1094" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1176", "results": [ "s:1093" ] }, { "annotation": null, "id": "r:1178", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1096" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 2.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1095" ] }, { "annotation": null, "id": "r:1179", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1096" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1178", "results": [ "s:1095" ] }, { "annotation": null, "id": "r:1180", "op": "linear.subtract", "operands": [ "s:1097", "s:1105" ], "params": {}, "parent": null, "results": [ "s:1096" ] }, { "annotation": null, "id": "r:1181", "op": "linear.add", "operands": [ "s:1097", "s:1104" ], "params": {}, "parent": "r:1180", "results": [ "s:1096" ] }, { "annotation": null, "id": "r:1182", "op": "linear.add_const", "operands": [ "s:1098" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1097" ] }, { "annotation": null, "id": "r:1183", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:1100", "s:1103" ], "params": {}, "parent": null, "results": [ "s:1098" ] }, { "annotation": null, "id": "r:1184", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1099" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1183", "results": [ "s:1098" ] }, { "annotation": null, "id": "r:1185", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1101" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1100" ] }, { "annotation": null, "id": "r:1186", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -2, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1101" ] }, { "annotation": null, "id": "r:1187", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:1103" ] }, { "annotation": null, "id": "r:1188", "op": "linear.negate", "operands": [ "s:1105" ], "params": {}, "parent": "r:1180", "results": [ "s:1104" ] }, { "annotation": null, "id": "r:1189", "op": "linear.bool_to_01", "operands": [ "s:1107" ], "params": {}, "parent": null, "results": [ "s:1105" ] }, { "annotation": null, "id": "r:1190", "op": "linear.multiply_const", "operands": [ "s:1106" ], "params": { "scalar": 0.5 }, "parent": "r:1189", "results": [ "s:1105" ] }, { "annotation": null, "id": "r:1191", "op": "linear.add_const", "operands": [ "s:1107" ], "params": { "scalar": 1.0 }, "parent": "r:1189", "results": [ "s:1106" ] }, { "annotation": null, "id": "r:1192", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:1108" ], "params": {}, "parent": null, "results": [ "s:1107" ] }, { "annotation": null, "id": "r:1193", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1108" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:1192", "results": [ "s:1107" ] }, { "annotation": null, "id": "r:1194", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1108" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1193", "results": [ "s:1107" ] }, { "annotation": null, "id": "r:1195", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1109" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": null, "results": [ "s:1108" ] }, { "annotation": null, "id": "r:1196", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1109" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1195", "results": [ "s:1108" ] }, { "annotation": null, "id": "r:1197", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1110" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1109" ] }, { "annotation": null, "id": "r:1198", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -11, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1110" ] }, { "annotation": null, "id": "r:1199", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1112" ] }, { "annotation": null, "id": "r:1200", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1114", "s:1037", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1113" ] }, { "annotation": null, "id": "r:1201", "op": "linear.bool_to_01", "operands": [ "s:1116" ], "params": {}, "parent": null, "results": [ "s:1114" ] }, { "annotation": null, "id": "r:1202", "op": "linear.multiply_const", "operands": [ "s:1115" ], "params": { "scalar": 0.5 }, "parent": "r:1201", "results": [ "s:1114" ] }, { "annotation": null, "id": "r:1203", "op": "linear.add_const", "operands": [ "s:1116" ], "params": { "scalar": 1.0 }, "parent": "r:1201", "results": [ "s:1115" ] }, { "annotation": null, "id": "r:1204", "op": "swiglu.map_select.in_range", "operands": [ "s:1096", "s:1118" ], "params": { "n_slots": 3 }, "parent": null, "results": [ "s:1116" ] }, { "annotation": null, "id": "r:1205", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1117" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1204", "results": [ "s:1116" ] }, { "annotation": null, "id": "r:1206", "op": "linear.add_const", "operands": [ "s:1096" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1118" ] }, { "annotation": null, "id": "r:1207", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:1121" ], "params": {}, "parent": "r:1153", "results": [ "s:1119" ] }, { "annotation": null, "id": "r:1208", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1120" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1207", "results": [ "s:1119" ] }, { "annotation": null, "id": "r:1209", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1122" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1121" ] }, { "annotation": null, "id": "r:1210", "op": "linear.bool_to_01", "operands": [ "s:1124" ], "params": {}, "parent": null, "results": [ "s:1122" ] }, { "annotation": null, "id": "r:1211", "op": "linear.multiply_const", "operands": [ "s:1123" ], "params": { "scalar": 0.5 }, "parent": "r:1210", "results": [ "s:1122" ] }, { "annotation": null, "id": "r:1212", "op": "linear.add_const", "operands": [ "s:1124" ], "params": { "scalar": 1.0 }, "parent": "r:1210", "results": [ "s:1123" ] }, { "annotation": null, "id": "r:1213", "op": "swiglu.map_select.in_range", "operands": [ "s:1126", "s:1137" ], "params": { "n_slots": 7 }, "parent": null, "results": [ "s:1124" ] }, { "annotation": null, "id": "r:1214", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1125" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 28 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 7 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 7 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1213", "results": [ "s:1124" ] }, { "annotation": null, "id": "r:1215", "op": "linear.add", "operands": [ "s:1127", "s:1128" ], "params": {}, "parent": null, "results": [ "s:1126" ] }, { "annotation": null, "id": "r:1216", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1127" ] }, { "annotation": null, "id": "r:1217", "op": "linear.bool_to_01", "operands": [ "s:1130" ], "params": {}, "parent": null, "results": [ "s:1128" ] }, { "annotation": null, "id": "r:1218", "op": "linear.multiply_const", "operands": [ "s:1129" ], "params": { "scalar": 0.5 }, "parent": "r:1217", "results": [ "s:1128" ] }, { "annotation": null, "id": "r:1219", "op": "linear.add_const", "operands": [ "s:1130" ], "params": { "scalar": 1.0 }, "parent": "r:1217", "results": [ "s:1129" ] }, { "annotation": null, "id": "r:1220", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1133", "s:1134" ], "params": {}, "parent": null, "results": [ "s:1130" ] }, { "annotation": null, "id": "r:1221", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1131" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1220", "results": [ "s:1130" ] }, { "annotation": null, "id": "r:1222", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1131" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1221", "results": [ "s:1130" ] }, { "annotation": null, "id": "r:1223", "op": "linear.sum_nodes", "operands": [ "s:1133", "s:1134" ], "params": { "max_fanout": null }, "parent": "r:1220", "results": [ "s:1131" ] }, { "annotation": null, "id": "r:1224", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1127" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1133" ] }, { "annotation": null, "id": "r:1225", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1127" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1224", "results": [ "s:1133" ] }, { "annotation": null, "id": "r:1226", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:1135" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1134" ] }, { "annotation": null, "id": "r:1227", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1135" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1226", "results": [ "s:1134" ] }, { "annotation": null, "id": "r:1228", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -5, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1135" ] }, { "annotation": null, "id": "r:1229", "op": "linear.add_const", "operands": [ "s:1126" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1137" ] }, { "annotation": null, "id": "r:1230", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:1140", "s:1146" ], "params": {}, "parent": null, "results": [ "s:1138" ] }, { "annotation": null, "id": "r:1231", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1139" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1230", "results": [ "s:1138" ] }, { "annotation": null, "id": "r:1232", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1143", "s:1144" ], "params": {}, "parent": null, "results": [ "s:1140" ] }, { "annotation": null, "id": "r:1233", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1141" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1232", "results": [ "s:1140" ] }, { "annotation": null, "id": "r:1234", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1141" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1233", "results": [ "s:1140" ] }, { "annotation": null, "id": "r:1235", "op": "linear.sum_nodes", "operands": [ "s:1143", "s:1144" ], "params": { "max_fanout": null }, "parent": "r:1232", "results": [ "s:1141" ] }, { "annotation": null, "id": "r:1236", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 15.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1143" ] }, { "annotation": null, "id": "r:1237", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1236", "results": [ "s:1143" ] }, { "annotation": null, "id": "r:1238", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1145" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -16.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1144" ] }, { "annotation": null, "id": "r:1239", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1145" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1238", "results": [ "s:1144" ] }, { "annotation": null, "id": "r:1240", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:1145" ] }, { "annotation": null, "id": "r:1241", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:1150", "s:1168", "s:1195" ], "params": {}, "parent": null, "results": [ "s:1146" ] }, { "annotation": null, "id": "r:1242", "op": "linear.sum_nodes", "operands": [ "s:1148", "s:1166", "s:1193" ], "params": { "max_fanout": null }, "parent": "r:1241", "results": [ "s:1146" ] }, { "annotation": null, "id": "r:1243", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:1150" ], "params": {}, "parent": "r:1241", "results": [ "s:1148" ] }, { "annotation": null, "id": "r:1244", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1149" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1243", "results": [ "s:1148" ] }, { "annotation": null, "id": "r:1245", "op": "swiglu.map_select.select", "operands": [ "s:1152", "s:998", "s:1160" ], "params": {}, "parent": null, "results": [ "s:1150" ] }, { "annotation": null, "id": "r:1246", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1151" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1245", "results": [ "s:1150" ] }, { "annotation": null, "id": "r:1247", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1153" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 3.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1152" ] }, { "annotation": null, "id": "r:1248", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1153" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1247", "results": [ "s:1152" ] }, { "annotation": null, "id": "r:1249", "op": "linear.add_const", "operands": [ "s:1154" ], "params": { "scalar": 2.0 }, "parent": null, "results": [ "s:1153" ] }, { "annotation": null, "id": "r:1250", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:1156", "s:1159" ], "params": {}, "parent": null, "results": [ "s:1154" ] }, { "annotation": null, "id": "r:1251", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1155" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1250", "results": [ "s:1154" ] }, { "annotation": null, "id": "r:1252", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1157" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1156" ] }, { "annotation": null, "id": "r:1253", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -3, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1157" ] }, { "annotation": null, "id": "r:1254", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:1159" ] }, { "annotation": null, "id": "r:1255", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1161", "s:1005", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1160" ] }, { "annotation": null, "id": "r:1256", "op": "linear.bool_to_01", "operands": [ "s:1163" ], "params": {}, "parent": null, "results": [ "s:1161" ] }, { "annotation": null, "id": "r:1257", "op": "linear.multiply_const", "operands": [ "s:1162" ], "params": { "scalar": 0.5 }, "parent": "r:1256", "results": [ "s:1161" ] }, { "annotation": null, "id": "r:1258", "op": "linear.add_const", "operands": [ "s:1163" ], "params": { "scalar": 1.0 }, "parent": "r:1256", "results": [ "s:1162" ] }, { "annotation": null, "id": "r:1259", "op": "swiglu.map_select.in_range", "operands": [ "s:1153", "s:1165" ], "params": { "n_slots": 4 }, "parent": null, "results": [ "s:1163" ] }, { "annotation": null, "id": "r:1260", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1164" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 16 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 16, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 16, 4 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1259", "results": [ "s:1163" ] }, { "annotation": null, "id": "r:1261", "op": "linear.add_const", "operands": [ "s:1153" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1165" ] }, { "annotation": null, "id": "r:1262", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:1168" ], "params": {}, "parent": "r:1241", "results": [ "s:1166" ] }, { "annotation": null, "id": "r:1263", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1167" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1262", "results": [ "s:1166" ] }, { "annotation": null, "id": "r:1264", "op": "swiglu.map_select.select", "operands": [ "s:1170", "s:1112", "s:1187" ], "params": {}, "parent": null, "results": [ "s:1168" ] }, { "annotation": null, "id": "r:1265", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1169" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1264", "results": [ "s:1168" ] }, { "annotation": null, "id": "r:1266", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1171" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 2.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1170" ] }, { "annotation": null, "id": "r:1267", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1171" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1266", "results": [ "s:1170" ] }, { "annotation": null, "id": "r:1268", "op": "linear.subtract", "operands": [ "s:1172", "s:1180" ], "params": {}, "parent": null, "results": [ "s:1171" ] }, { "annotation": null, "id": "r:1269", "op": "linear.add", "operands": [ "s:1172", "s:1179" ], "params": {}, "parent": "r:1268", "results": [ "s:1171" ] }, { "annotation": null, "id": "r:1270", "op": "linear.add_const", "operands": [ "s:1173" ], "params": { "scalar": 2.0 }, "parent": null, "results": [ "s:1172" ] }, { "annotation": null, "id": "r:1271", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:1175", "s:1178" ], "params": {}, "parent": null, "results": [ "s:1173" ] }, { "annotation": null, "id": "r:1272", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1174" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1271", "results": [ "s:1173" ] }, { "annotation": null, "id": "r:1273", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1176" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1175" ] }, { "annotation": null, "id": "r:1274", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -3, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1176" ] }, { "annotation": null, "id": "r:1275", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:1178" ] }, { "annotation": null, "id": "r:1276", "op": "linear.negate", "operands": [ "s:1180" ], "params": {}, "parent": "r:1268", "results": [ "s:1179" ] }, { "annotation": null, "id": "r:1277", "op": "linear.bool_to_01", "operands": [ "s:1182" ], "params": {}, "parent": null, "results": [ "s:1180" ] }, { "annotation": null, "id": "r:1278", "op": "linear.multiply_const", "operands": [ "s:1181" ], "params": { "scalar": 0.5 }, "parent": "r:1277", "results": [ "s:1180" ] }, { "annotation": null, "id": "r:1279", "op": "linear.add_const", "operands": [ "s:1182" ], "params": { "scalar": 1.0 }, "parent": "r:1277", "results": [ "s:1181" ] }, { "annotation": null, "id": "r:1280", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:1183" ], "params": {}, "parent": null, "results": [ "s:1182" ] }, { "annotation": null, "id": "r:1281", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1183" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:1280", "results": [ "s:1182" ] }, { "annotation": null, "id": "r:1282", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1183" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1281", "results": [ "s:1182" ] }, { "annotation": null, "id": "r:1283", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1184" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": null, "results": [ "s:1183" ] }, { "annotation": null, "id": "r:1284", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1184" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1283", "results": [ "s:1183" ] }, { "annotation": null, "id": "r:1285", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1185" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1184" ] }, { "annotation": null, "id": "r:1286", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -12, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1185" ] }, { "annotation": null, "id": "r:1287", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1188", "s:1037", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1187" ] }, { "annotation": null, "id": "r:1288", "op": "linear.bool_to_01", "operands": [ "s:1190" ], "params": {}, "parent": null, "results": [ "s:1188" ] }, { "annotation": null, "id": "r:1289", "op": "linear.multiply_const", "operands": [ "s:1189" ], "params": { "scalar": 0.5 }, "parent": "r:1288", "results": [ "s:1188" ] }, { "annotation": null, "id": "r:1290", "op": "linear.add_const", "operands": [ "s:1190" ], "params": { "scalar": 1.0 }, "parent": "r:1288", "results": [ "s:1189" ] }, { "annotation": null, "id": "r:1291", "op": "swiglu.map_select.in_range", "operands": [ "s:1171", "s:1192" ], "params": { "n_slots": 3 }, "parent": null, "results": [ "s:1190" ] }, { "annotation": null, "id": "r:1292", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1191" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1291", "results": [ "s:1190" ] }, { "annotation": null, "id": "r:1293", "op": "linear.add_const", "operands": [ "s:1171" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1192" ] }, { "annotation": null, "id": "r:1294", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:1195" ], "params": {}, "parent": "r:1241", "results": [ "s:1193" ] }, { "annotation": null, "id": "r:1295", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1194" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1294", "results": [ "s:1193" ] }, { "annotation": null, "id": "r:1296", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1196" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1195" ] }, { "annotation": null, "id": "r:1297", "op": "linear.bool_to_01", "operands": [ "s:1198" ], "params": {}, "parent": null, "results": [ "s:1196" ] }, { "annotation": null, "id": "r:1298", "op": "linear.multiply_const", "operands": [ "s:1197" ], "params": { "scalar": 0.5 }, "parent": "r:1297", "results": [ "s:1196" ] }, { "annotation": null, "id": "r:1299", "op": "linear.add_const", "operands": [ "s:1198" ], "params": { "scalar": 1.0 }, "parent": "r:1297", "results": [ "s:1197" ] }, { "annotation": null, "id": "r:1300", "op": "swiglu.map_select.in_range", "operands": [ "s:1200", "s:1211" ], "params": { "n_slots": 7 }, "parent": null, "results": [ "s:1198" ] }, { "annotation": null, "id": "r:1301", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1199" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 28 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 7 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 7 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1300", "results": [ "s:1198" ] }, { "annotation": null, "id": "r:1302", "op": "linear.add", "operands": [ "s:1201", "s:1202" ], "params": {}, "parent": null, "results": [ "s:1200" ] }, { "annotation": null, "id": "r:1303", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1201" ] }, { "annotation": null, "id": "r:1304", "op": "linear.bool_to_01", "operands": [ "s:1204" ], "params": {}, "parent": null, "results": [ "s:1202" ] }, { "annotation": null, "id": "r:1305", "op": "linear.multiply_const", "operands": [ "s:1203" ], "params": { "scalar": 0.5 }, "parent": "r:1304", "results": [ "s:1202" ] }, { "annotation": null, "id": "r:1306", "op": "linear.add_const", "operands": [ "s:1204" ], "params": { "scalar": 1.0 }, "parent": "r:1304", "results": [ "s:1203" ] }, { "annotation": null, "id": "r:1307", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1207", "s:1208" ], "params": {}, "parent": null, "results": [ "s:1204" ] }, { "annotation": null, "id": "r:1308", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1205" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1307", "results": [ "s:1204" ] }, { "annotation": null, "id": "r:1309", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1205" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1308", "results": [ "s:1204" ] }, { "annotation": null, "id": "r:1310", "op": "linear.sum_nodes", "operands": [ "s:1207", "s:1208" ], "params": { "max_fanout": null }, "parent": "r:1307", "results": [ "s:1205" ] }, { "annotation": null, "id": "r:1311", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1201" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 2.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1207" ] }, { "annotation": null, "id": "r:1312", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1201" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1311", "results": [ "s:1207" ] }, { "annotation": null, "id": "r:1313", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:1209" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1208" ] }, { "annotation": null, "id": "r:1314", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1209" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1313", "results": [ "s:1208" ] }, { "annotation": null, "id": "r:1315", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -5, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1209" ] }, { "annotation": null, "id": "r:1316", "op": "linear.add_const", "operands": [ "s:1200" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1211" ] }, { "annotation": null, "id": "r:1317", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:1214", "s:1220" ], "params": {}, "parent": null, "results": [ "s:1212" ] }, { "annotation": null, "id": "r:1318", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1213" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1317", "results": [ "s:1212" ] }, { "annotation": null, "id": "r:1319", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1217", "s:1218" ], "params": {}, "parent": null, "results": [ "s:1214" ] }, { "annotation": null, "id": "r:1320", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1215" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1319", "results": [ "s:1214" ] }, { "annotation": null, "id": "r:1321", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1215" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1320", "results": [ "s:1214" ] }, { "annotation": null, "id": "r:1322", "op": "linear.sum_nodes", "operands": [ "s:1217", "s:1218" ], "params": { "max_fanout": null }, "parent": "r:1319", "results": [ "s:1215" ] }, { "annotation": null, "id": "r:1323", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 16.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1217" ] }, { "annotation": null, "id": "r:1324", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1323", "results": [ "s:1217" ] }, { "annotation": null, "id": "r:1325", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1219" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -17.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1218" ] }, { "annotation": null, "id": "r:1326", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1219" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1325", "results": [ "s:1218" ] }, { "annotation": null, "id": "r:1327", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:1219" ] }, { "annotation": null, "id": "r:1328", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:1224", "s:1242", "s:1269" ], "params": {}, "parent": null, "results": [ "s:1220" ] }, { "annotation": null, "id": "r:1329", "op": "linear.sum_nodes", "operands": [ "s:1222", "s:1240", "s:1267" ], "params": { "max_fanout": null }, "parent": "r:1328", "results": [ "s:1220" ] }, { "annotation": null, "id": "r:1330", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:1224" ], "params": {}, "parent": "r:1328", "results": [ "s:1222" ] }, { "annotation": null, "id": "r:1331", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1223" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1330", "results": [ "s:1222" ] }, { "annotation": null, "id": "r:1332", "op": "swiglu.map_select.select", "operands": [ "s:1226", "s:998", "s:1234" ], "params": {}, "parent": null, "results": [ "s:1224" ] }, { "annotation": null, "id": "r:1333", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1225" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1332", "results": [ "s:1224" ] }, { "annotation": null, "id": "r:1334", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1227" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 3.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1226" ] }, { "annotation": null, "id": "r:1335", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1227" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1334", "results": [ "s:1226" ] }, { "annotation": null, "id": "r:1336", "op": "linear.add_const", "operands": [ "s:1228" ], "params": { "scalar": 3.0 }, "parent": null, "results": [ "s:1227" ] }, { "annotation": null, "id": "r:1337", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:1230", "s:1233" ], "params": {}, "parent": null, "results": [ "s:1228" ] }, { "annotation": null, "id": "r:1338", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1229" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1337", "results": [ "s:1228" ] }, { "annotation": null, "id": "r:1339", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1231" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1230" ] }, { "annotation": null, "id": "r:1340", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -4, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1231" ] }, { "annotation": null, "id": "r:1341", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:1233" ] }, { "annotation": null, "id": "r:1342", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1235", "s:1005", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1234" ] }, { "annotation": null, "id": "r:1343", "op": "linear.bool_to_01", "operands": [ "s:1237" ], "params": {}, "parent": null, "results": [ "s:1235" ] }, { "annotation": null, "id": "r:1344", "op": "linear.multiply_const", "operands": [ "s:1236" ], "params": { "scalar": 0.5 }, "parent": "r:1343", "results": [ "s:1235" ] }, { "annotation": null, "id": "r:1345", "op": "linear.add_const", "operands": [ "s:1237" ], "params": { "scalar": 1.0 }, "parent": "r:1343", "results": [ "s:1236" ] }, { "annotation": null, "id": "r:1346", "op": "swiglu.map_select.in_range", "operands": [ "s:1227", "s:1239" ], "params": { "n_slots": 4 }, "parent": null, "results": [ "s:1237" ] }, { "annotation": null, "id": "r:1347", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1238" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 16 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 16, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 16, 4 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1346", "results": [ "s:1237" ] }, { "annotation": null, "id": "r:1348", "op": "linear.add_const", "operands": [ "s:1227" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1239" ] }, { "annotation": null, "id": "r:1349", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:1242" ], "params": {}, "parent": "r:1328", "results": [ "s:1240" ] }, { "annotation": null, "id": "r:1350", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1241" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1349", "results": [ "s:1240" ] }, { "annotation": null, "id": "r:1351", "op": "swiglu.map_select.select", "operands": [ "s:1244", "s:1112", "s:1261" ], "params": {}, "parent": null, "results": [ "s:1242" ] }, { "annotation": null, "id": "r:1352", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1243" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1351", "results": [ "s:1242" ] }, { "annotation": null, "id": "r:1353", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1245" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 2.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1244" ] }, { "annotation": null, "id": "r:1354", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1245" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1353", "results": [ "s:1244" ] }, { "annotation": null, "id": "r:1355", "op": "linear.subtract", "operands": [ "s:1246", "s:1254" ], "params": {}, "parent": null, "results": [ "s:1245" ] }, { "annotation": null, "id": "r:1356", "op": "linear.add", "operands": [ "s:1246", "s:1253" ], "params": {}, "parent": "r:1355", "results": [ "s:1245" ] }, { "annotation": null, "id": "r:1357", "op": "linear.add_const", "operands": [ "s:1247" ], "params": { "scalar": 3.0 }, "parent": null, "results": [ "s:1246" ] }, { "annotation": null, "id": "r:1358", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:1249", "s:1252" ], "params": {}, "parent": null, "results": [ "s:1247" ] }, { "annotation": null, "id": "r:1359", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1248" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1358", "results": [ "s:1247" ] }, { "annotation": null, "id": "r:1360", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1250" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1249" ] }, { "annotation": null, "id": "r:1361", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -4, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1250" ] }, { "annotation": null, "id": "r:1362", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:1252" ] }, { "annotation": null, "id": "r:1363", "op": "linear.negate", "operands": [ "s:1254" ], "params": {}, "parent": "r:1355", "results": [ "s:1253" ] }, { "annotation": null, "id": "r:1364", "op": "linear.bool_to_01", "operands": [ "s:1256" ], "params": {}, "parent": null, "results": [ "s:1254" ] }, { "annotation": null, "id": "r:1365", "op": "linear.multiply_const", "operands": [ "s:1255" ], "params": { "scalar": 0.5 }, "parent": "r:1364", "results": [ "s:1254" ] }, { "annotation": null, "id": "r:1366", "op": "linear.add_const", "operands": [ "s:1256" ], "params": { "scalar": 1.0 }, "parent": "r:1364", "results": [ "s:1255" ] }, { "annotation": null, "id": "r:1367", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:1257" ], "params": {}, "parent": null, "results": [ "s:1256" ] }, { "annotation": null, "id": "r:1368", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1257" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:1367", "results": [ "s:1256" ] }, { "annotation": null, "id": "r:1369", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1257" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1368", "results": [ "s:1256" ] }, { "annotation": null, "id": "r:1370", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1258" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": null, "results": [ "s:1257" ] }, { "annotation": null, "id": "r:1371", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1258" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1370", "results": [ "s:1257" ] }, { "annotation": null, "id": "r:1372", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1259" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1258" ] }, { "annotation": null, "id": "r:1373", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -13, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1259" ] }, { "annotation": null, "id": "r:1374", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1262", "s:1037", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1261" ] }, { "annotation": null, "id": "r:1375", "op": "linear.bool_to_01", "operands": [ "s:1264" ], "params": {}, "parent": null, "results": [ "s:1262" ] }, { "annotation": null, "id": "r:1376", "op": "linear.multiply_const", "operands": [ "s:1263" ], "params": { "scalar": 0.5 }, "parent": "r:1375", "results": [ "s:1262" ] }, { "annotation": null, "id": "r:1377", "op": "linear.add_const", "operands": [ "s:1264" ], "params": { "scalar": 1.0 }, "parent": "r:1375", "results": [ "s:1263" ] }, { "annotation": null, "id": "r:1378", "op": "swiglu.map_select.in_range", "operands": [ "s:1245", "s:1266" ], "params": { "n_slots": 3 }, "parent": null, "results": [ "s:1264" ] }, { "annotation": null, "id": "r:1379", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1265" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1378", "results": [ "s:1264" ] }, { "annotation": null, "id": "r:1380", "op": "linear.add_const", "operands": [ "s:1245" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1266" ] }, { "annotation": null, "id": "r:1381", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:1269" ], "params": {}, "parent": "r:1328", "results": [ "s:1267" ] }, { "annotation": null, "id": "r:1382", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1268" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1381", "results": [ "s:1267" ] }, { "annotation": null, "id": "r:1383", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1270" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1269" ] }, { "annotation": null, "id": "r:1384", "op": "linear.bool_to_01", "operands": [ "s:1272" ], "params": {}, "parent": null, "results": [ "s:1270" ] }, { "annotation": null, "id": "r:1385", "op": "linear.multiply_const", "operands": [ "s:1271" ], "params": { "scalar": 0.5 }, "parent": "r:1384", "results": [ "s:1270" ] }, { "annotation": null, "id": "r:1386", "op": "linear.add_const", "operands": [ "s:1272" ], "params": { "scalar": 1.0 }, "parent": "r:1384", "results": [ "s:1271" ] }, { "annotation": null, "id": "r:1387", "op": "swiglu.map_select.in_range", "operands": [ "s:1274", "s:1285" ], "params": { "n_slots": 7 }, "parent": null, "results": [ "s:1272" ] }, { "annotation": null, "id": "r:1388", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1273" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 28 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 7 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 7 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1387", "results": [ "s:1272" ] }, { "annotation": null, "id": "r:1389", "op": "linear.add", "operands": [ "s:1275", "s:1276" ], "params": {}, "parent": null, "results": [ "s:1274" ] }, { "annotation": null, "id": "r:1390", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1275" ] }, { "annotation": null, "id": "r:1391", "op": "linear.bool_to_01", "operands": [ "s:1278" ], "params": {}, "parent": null, "results": [ "s:1276" ] }, { "annotation": null, "id": "r:1392", "op": "linear.multiply_const", "operands": [ "s:1277" ], "params": { "scalar": 0.5 }, "parent": "r:1391", "results": [ "s:1276" ] }, { "annotation": null, "id": "r:1393", "op": "linear.add_const", "operands": [ "s:1278" ], "params": { "scalar": 1.0 }, "parent": "r:1391", "results": [ "s:1277" ] }, { "annotation": null, "id": "r:1394", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1281", "s:1282" ], "params": {}, "parent": null, "results": [ "s:1278" ] }, { "annotation": null, "id": "r:1395", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1279" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1394", "results": [ "s:1278" ] }, { "annotation": null, "id": "r:1396", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1279" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1395", "results": [ "s:1278" ] }, { "annotation": null, "id": "r:1397", "op": "linear.sum_nodes", "operands": [ "s:1281", "s:1282" ], "params": { "max_fanout": null }, "parent": "r:1394", "results": [ "s:1279" ] }, { "annotation": null, "id": "r:1398", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1275" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 3.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1281" ] }, { "annotation": null, "id": "r:1399", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1275" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1398", "results": [ "s:1281" ] }, { "annotation": null, "id": "r:1400", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:1283" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1282" ] }, { "annotation": null, "id": "r:1401", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1283" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1400", "results": [ "s:1282" ] }, { "annotation": null, "id": "r:1402", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -5, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1283" ] }, { "annotation": null, "id": "r:1403", "op": "linear.add_const", "operands": [ "s:1274" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1285" ] }, { "annotation": null, "id": "r:1404", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:1288", "s:1294" ], "params": {}, "parent": null, "results": [ "s:1286" ] }, { "annotation": null, "id": "r:1405", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1287" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1404", "results": [ "s:1286" ] }, { "annotation": null, "id": "r:1406", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1291", "s:1292" ], "params": {}, "parent": null, "results": [ "s:1288" ] }, { "annotation": null, "id": "r:1407", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1289" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1406", "results": [ "s:1288" ] }, { "annotation": null, "id": "r:1408", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1289" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1407", "results": [ "s:1288" ] }, { "annotation": null, "id": "r:1409", "op": "linear.sum_nodes", "operands": [ "s:1291", "s:1292" ], "params": { "max_fanout": null }, "parent": "r:1406", "results": [ "s:1289" ] }, { "annotation": null, "id": "r:1410", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 17.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1291" ] }, { "annotation": null, "id": "r:1411", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1410", "results": [ "s:1291" ] }, { "annotation": null, "id": "r:1412", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1293" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -18.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1292" ] }, { "annotation": null, "id": "r:1413", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1293" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1412", "results": [ "s:1292" ] }, { "annotation": null, "id": "r:1414", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:1293" ] }, { "annotation": null, "id": "r:1415", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:1298", "s:1301", "s:1304" ], "params": {}, "parent": null, "results": [ "s:1294" ] }, { "annotation": null, "id": "r:1416", "op": "linear.sum_nodes", "operands": [ "s:1296", "s:1299", "s:1302" ], "params": { "max_fanout": null }, "parent": "r:1415", "results": [ "s:1294" ] }, { "annotation": null, "id": "r:1417", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:1298" ], "params": {}, "parent": "r:1415", "results": [ "s:1296" ] }, { "annotation": null, "id": "r:1418", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1297" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1417", "results": [ "s:1296" ] }, { "annotation": null, "id": "r:1419", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1298" ] }, { "annotation": null, "id": "r:1420", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:1301" ], "params": {}, "parent": "r:1415", "results": [ "s:1299" ] }, { "annotation": null, "id": "r:1421", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1300" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1420", "results": [ "s:1299" ] }, { "annotation": null, "id": "r:1422", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1301" ] }, { "annotation": null, "id": "r:1423", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:1304" ], "params": {}, "parent": "r:1415", "results": [ "s:1302" ] }, { "annotation": null, "id": "r:1424", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1303" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1423", "results": [ "s:1302" ] }, { "annotation": null, "id": "r:1425", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1305" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1304" ] }, { "annotation": null, "id": "r:1426", "op": "linear.bool_to_01", "operands": [ "s:1307" ], "params": {}, "parent": null, "results": [ "s:1305" ] }, { "annotation": null, "id": "r:1427", "op": "linear.multiply_const", "operands": [ "s:1306" ], "params": { "scalar": 0.5 }, "parent": "r:1426", "results": [ "s:1305" ] }, { "annotation": null, "id": "r:1428", "op": "linear.add_const", "operands": [ "s:1307" ], "params": { "scalar": 1.0 }, "parent": "r:1426", "results": [ "s:1306" ] }, { "annotation": null, "id": "r:1429", "op": "swiglu.map_select.in_range", "operands": [ "s:1309", "s:1320" ], "params": { "n_slots": 7 }, "parent": null, "results": [ "s:1307" ] }, { "annotation": null, "id": "r:1430", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1308" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 28 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 7 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 28, 7 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1429", "results": [ "s:1307" ] }, { "annotation": null, "id": "r:1431", "op": "linear.add", "operands": [ "s:1310", "s:1311" ], "params": {}, "parent": null, "results": [ "s:1309" ] }, { "annotation": null, "id": "r:1432", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1310" ] }, { "annotation": null, "id": "r:1433", "op": "linear.bool_to_01", "operands": [ "s:1313" ], "params": {}, "parent": null, "results": [ "s:1311" ] }, { "annotation": null, "id": "r:1434", "op": "linear.multiply_const", "operands": [ "s:1312" ], "params": { "scalar": 0.5 }, "parent": "r:1433", "results": [ "s:1311" ] }, { "annotation": null, "id": "r:1435", "op": "linear.add_const", "operands": [ "s:1313" ], "params": { "scalar": 1.0 }, "parent": "r:1433", "results": [ "s:1312" ] }, { "annotation": null, "id": "r:1436", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1316", "s:1317" ], "params": {}, "parent": null, "results": [ "s:1313" ] }, { "annotation": null, "id": "r:1437", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1314" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1436", "results": [ "s:1313" ] }, { "annotation": null, "id": "r:1438", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1314" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1437", "results": [ "s:1313" ] }, { "annotation": null, "id": "r:1439", "op": "linear.sum_nodes", "operands": [ "s:1316", "s:1317" ], "params": { "max_fanout": null }, "parent": "r:1436", "results": [ "s:1314" ] }, { "annotation": null, "id": "r:1440", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1310" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 4.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1316" ] }, { "annotation": null, "id": "r:1441", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1310" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1440", "results": [ "s:1316" ] }, { "annotation": null, "id": "r:1442", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:1318" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1317" ] }, { "annotation": null, "id": "r:1443", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1318" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 33 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1442", "results": [ "s:1317" ] }, { "annotation": null, "id": "r:1444", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -5, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1318" ] }, { "annotation": null, "id": "r:1445", "op": "linear.add_const", "operands": [ "s:1309" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1320" ] }, { "annotation": null, "id": "r:1446", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:1323", "s:1329" ], "params": {}, "parent": null, "results": [ "s:1321" ] }, { "annotation": null, "id": "r:1447", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1322" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1446", "results": [ "s:1321" ] }, { "annotation": null, "id": "r:1448", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1326", "s:1327" ], "params": {}, "parent": null, "results": [ "s:1323" ] }, { "annotation": null, "id": "r:1449", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1324" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1448", "results": [ "s:1323" ] }, { "annotation": null, "id": "r:1450", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1324" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1449", "results": [ "s:1323" ] }, { "annotation": null, "id": "r:1451", "op": "linear.sum_nodes", "operands": [ "s:1326", "s:1327" ], "params": { "max_fanout": null }, "parent": "r:1448", "results": [ "s:1324" ] }, { "annotation": null, "id": "r:1452", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 18.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1326" ] }, { "annotation": null, "id": "r:1453", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1452", "results": [ "s:1326" ] }, { "annotation": null, "id": "r:1454", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1328" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -19.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1327" ] }, { "annotation": null, "id": "r:1455", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1328" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1454", "results": [ "s:1327" ] }, { "annotation": null, "id": "r:1456", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:1328" ] }, { "annotation": null, "id": "r:1457", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:1333", "s:1336", "s:1339" ], "params": {}, "parent": null, "results": [ "s:1329" ] }, { "annotation": null, "id": "r:1458", "op": "linear.sum_nodes", "operands": [ "s:1331", "s:1334", "s:1337" ], "params": { "max_fanout": null }, "parent": "r:1457", "results": [ "s:1329" ] }, { "annotation": null, "id": "r:1459", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:1333" ], "params": {}, "parent": "r:1457", "results": [ "s:1331" ] }, { "annotation": null, "id": "r:1460", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1332" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1459", "results": [ "s:1331" ] }, { "annotation": null, "id": "r:1461", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1333" ] }, { "annotation": null, "id": "r:1462", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:1336" ], "params": {}, "parent": "r:1457", "results": [ "s:1334" ] }, { "annotation": null, "id": "r:1463", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1335" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1462", "results": [ "s:1334" ] }, { "annotation": null, "id": "r:1464", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1336" ] }, { "annotation": null, "id": "r:1465", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:1339" ], "params": {}, "parent": "r:1457", "results": [ "s:1337" ] }, { "annotation": null, "id": "r:1466", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1338" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1465", "results": [ "s:1337" ] }, { "annotation": null, "id": "r:1467", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1339" ] }, { "annotation": null, "id": "r:1468", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:1342", "s:1348" ], "params": {}, "parent": null, "results": [ "s:1340" ] }, { "annotation": null, "id": "r:1469", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1341" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1468", "results": [ "s:1340" ] }, { "annotation": null, "id": "r:1470", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1345", "s:1346" ], "params": {}, "parent": null, "results": [ "s:1342" ] }, { "annotation": null, "id": "r:1471", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1343" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1470", "results": [ "s:1342" ] }, { "annotation": null, "id": "r:1472", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1343" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1471", "results": [ "s:1342" ] }, { "annotation": null, "id": "r:1473", "op": "linear.sum_nodes", "operands": [ "s:1345", "s:1346" ], "params": { "max_fanout": null }, "parent": "r:1470", "results": [ "s:1343" ] }, { "annotation": null, "id": "r:1474", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 19.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1345" ] }, { "annotation": null, "id": "r:1475", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1474", "results": [ "s:1345" ] }, { "annotation": null, "id": "r:1476", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1347" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -20.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1346" ] }, { "annotation": null, "id": "r:1477", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1347" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1476", "results": [ "s:1346" ] }, { "annotation": null, "id": "r:1478", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:1347" ] }, { "annotation": null, "id": "r:1479", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:1352", "s:1355", "s:1358" ], "params": {}, "parent": null, "results": [ "s:1348" ] }, { "annotation": null, "id": "r:1480", "op": "linear.sum_nodes", "operands": [ "s:1350", "s:1353", "s:1356" ], "params": { "max_fanout": null }, "parent": "r:1479", "results": [ "s:1348" ] }, { "annotation": null, "id": "r:1481", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:1352" ], "params": {}, "parent": "r:1479", "results": [ "s:1350" ] }, { "annotation": null, "id": "r:1482", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1351" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1481", "results": [ "s:1350" ] }, { "annotation": null, "id": "r:1483", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1352" ] }, { "annotation": null, "id": "r:1484", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:1355" ], "params": {}, "parent": "r:1479", "results": [ "s:1353" ] }, { "annotation": null, "id": "r:1485", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1354" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1484", "results": [ "s:1353" ] }, { "annotation": null, "id": "r:1486", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1355" ] }, { "annotation": null, "id": "r:1487", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:1358" ], "params": {}, "parent": "r:1479", "results": [ "s:1356" ] }, { "annotation": null, "id": "r:1488", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1357" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1487", "results": [ "s:1356" ] }, { "annotation": null, "id": "r:1489", "op": "swiglu.map_select.select", "operands": [ "s:1360", "s:1368", "s:1369" ], "params": {}, "parent": null, "results": [ "s:1358" ] }, { "annotation": null, "id": "r:1490", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1359" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1489", "results": [ "s:1358" ] }, { "annotation": null, "id": "r:1491", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1361" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 5.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1360" ] }, { "annotation": null, "id": "r:1492", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1361" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1491", "results": [ "s:1360" ] }, { "annotation": null, "id": "r:1493", "op": "linear.add_const", "operands": [ "s:1362" ], "params": { "scalar": 0.0 }, "parent": null, "results": [ "s:1361" ] }, { "annotation": null, "id": "r:1494", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:1364", "s:1367" ], "params": {}, "parent": null, "results": [ "s:1362" ] }, { "annotation": null, "id": "r:1495", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1363" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1494", "results": [ "s:1362" ] }, { "annotation": null, "id": "r:1496", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1365" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1364" ] }, { "annotation": null, "id": "r:1497", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -1, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1365" ] }, { "annotation": null, "id": "r:1498", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:1367" ] }, { "annotation": null, "id": "r:1499", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1368" ] }, { "annotation": null, "id": "r:1500", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1370", "s:1375", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1369" ] }, { "annotation": null, "id": "r:1501", "op": "linear.bool_to_01", "operands": [ "s:1372" ], "params": {}, "parent": null, "results": [ "s:1370" ] }, { "annotation": null, "id": "r:1502", "op": "linear.multiply_const", "operands": [ "s:1371" ], "params": { "scalar": 0.5 }, "parent": "r:1501", "results": [ "s:1370" ] }, { "annotation": null, "id": "r:1503", "op": "linear.add_const", "operands": [ "s:1372" ], "params": { "scalar": 1.0 }, "parent": "r:1501", "results": [ "s:1371" ] }, { "annotation": null, "id": "r:1504", "op": "swiglu.map_select.in_range", "operands": [ "s:1361", "s:1374" ], "params": { "n_slots": 6 }, "parent": null, "results": [ "s:1372" ] }, { "annotation": null, "id": "r:1505", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1373" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 24 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 6 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 6 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1504", "results": [ "s:1372" ] }, { "annotation": null, "id": "r:1506", "op": "linear.add_const", "operands": [ "s:1361" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1374" ] }, { "annotation": null, "id": "r:1507", "op": "linear.bool_to_01", "operands": [ "s:1377" ], "params": {}, "parent": null, "results": [ "s:1375" ] }, { "annotation": null, "id": "r:1508", "op": "linear.multiply_const", "operands": [ "s:1376" ], "params": { "scalar": 0.5 }, "parent": "r:1507", "results": [ "s:1375" ] }, { "annotation": null, "id": "r:1509", "op": "linear.add_const", "operands": [ "s:1377" ], "params": { "scalar": 1.0 }, "parent": "r:1507", "results": [ "s:1376" ] }, { "annotation": null, "id": "r:1510", "op": "swiglu.map_select.in_range", "operands": [ "s:1379", "s:1380" ], "params": { "n_slots": 6 }, "parent": null, "results": [ "s:1377" ] }, { "annotation": null, "id": "r:1511", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1378" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 24 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 6 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 6 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1510", "results": [ "s:1377" ] }, { "annotation": null, "id": "r:1512", "op": "linear.add_const", "operands": [ "s:18" ], "params": { "scalar": -8.0 }, "parent": null, "results": [ "s:1379" ] }, { "annotation": null, "id": "r:1513", "op": "linear.add_const", "operands": [ "s:1379" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1380" ] }, { "annotation": null, "id": "r:1514", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:1383", "s:1389" ], "params": {}, "parent": null, "results": [ "s:1381" ] }, { "annotation": null, "id": "r:1515", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1382" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1514", "results": [ "s:1381" ] }, { "annotation": null, "id": "r:1516", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1386", "s:1387" ], "params": {}, "parent": null, "results": [ "s:1383" ] }, { "annotation": null, "id": "r:1517", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1384" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1516", "results": [ "s:1383" ] }, { "annotation": null, "id": "r:1518", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1384" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1517", "results": [ "s:1383" ] }, { "annotation": null, "id": "r:1519", "op": "linear.sum_nodes", "operands": [ "s:1386", "s:1387" ], "params": { "max_fanout": null }, "parent": "r:1516", "results": [ "s:1384" ] }, { "annotation": null, "id": "r:1520", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 20.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1386" ] }, { "annotation": null, "id": "r:1521", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1520", "results": [ "s:1386" ] }, { "annotation": null, "id": "r:1522", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1388" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -21.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1387" ] }, { "annotation": null, "id": "r:1523", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1388" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1522", "results": [ "s:1387" ] }, { "annotation": null, "id": "r:1524", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:1388" ] }, { "annotation": null, "id": "r:1525", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:1393", "s:1396", "s:1399" ], "params": {}, "parent": null, "results": [ "s:1389" ] }, { "annotation": null, "id": "r:1526", "op": "linear.sum_nodes", "operands": [ "s:1391", "s:1394", "s:1397" ], "params": { "max_fanout": null }, "parent": "r:1525", "results": [ "s:1389" ] }, { "annotation": null, "id": "r:1527", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:1393" ], "params": {}, "parent": "r:1525", "results": [ "s:1391" ] }, { "annotation": null, "id": "r:1528", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1392" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1527", "results": [ "s:1391" ] }, { "annotation": null, "id": "r:1529", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1393" ] }, { "annotation": null, "id": "r:1530", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:1396" ], "params": {}, "parent": "r:1525", "results": [ "s:1394" ] }, { "annotation": null, "id": "r:1531", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1395" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1530", "results": [ "s:1394" ] }, { "annotation": null, "id": "r:1532", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1396" ] }, { "annotation": null, "id": "r:1533", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:1399" ], "params": {}, "parent": "r:1525", "results": [ "s:1397" ] }, { "annotation": null, "id": "r:1534", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1398" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1533", "results": [ "s:1397" ] }, { "annotation": null, "id": "r:1535", "op": "swiglu.map_select.select", "operands": [ "s:1401", "s:1368", "s:1409" ], "params": {}, "parent": null, "results": [ "s:1399" ] }, { "annotation": null, "id": "r:1536", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1400" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1535", "results": [ "s:1399" ] }, { "annotation": null, "id": "r:1537", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1402" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 5.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1401" ] }, { "annotation": null, "id": "r:1538", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1402" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1537", "results": [ "s:1401" ] }, { "annotation": null, "id": "r:1539", "op": "linear.add_const", "operands": [ "s:1403" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1402" ] }, { "annotation": null, "id": "r:1540", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:1405", "s:1408" ], "params": {}, "parent": null, "results": [ "s:1403" ] }, { "annotation": null, "id": "r:1541", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1404" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1540", "results": [ "s:1403" ] }, { "annotation": null, "id": "r:1542", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1406" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1405" ] }, { "annotation": null, "id": "r:1543", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -2, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1406" ] }, { "annotation": null, "id": "r:1544", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:1408" ] }, { "annotation": null, "id": "r:1545", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1410", "s:1375", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1409" ] }, { "annotation": null, "id": "r:1546", "op": "linear.bool_to_01", "operands": [ "s:1412" ], "params": {}, "parent": null, "results": [ "s:1410" ] }, { "annotation": null, "id": "r:1547", "op": "linear.multiply_const", "operands": [ "s:1411" ], "params": { "scalar": 0.5 }, "parent": "r:1546", "results": [ "s:1410" ] }, { "annotation": null, "id": "r:1548", "op": "linear.add_const", "operands": [ "s:1412" ], "params": { "scalar": 1.0 }, "parent": "r:1546", "results": [ "s:1411" ] }, { "annotation": null, "id": "r:1549", "op": "swiglu.map_select.in_range", "operands": [ "s:1402", "s:1414" ], "params": { "n_slots": 6 }, "parent": null, "results": [ "s:1412" ] }, { "annotation": null, "id": "r:1550", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1413" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 24 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 6 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 6 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1549", "results": [ "s:1412" ] }, { "annotation": null, "id": "r:1551", "op": "linear.add_const", "operands": [ "s:1402" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1414" ] }, { "annotation": null, "id": "r:1552", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:1417", "s:1423" ], "params": {}, "parent": null, "results": [ "s:1415" ] }, { "annotation": null, "id": "r:1553", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1416" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1552", "results": [ "s:1415" ] }, { "annotation": null, "id": "r:1554", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1420", "s:1421" ], "params": {}, "parent": null, "results": [ "s:1417" ] }, { "annotation": null, "id": "r:1555", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1418" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1554", "results": [ "s:1417" ] }, { "annotation": null, "id": "r:1556", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1418" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1555", "results": [ "s:1417" ] }, { "annotation": null, "id": "r:1557", "op": "linear.sum_nodes", "operands": [ "s:1420", "s:1421" ], "params": { "max_fanout": null }, "parent": "r:1554", "results": [ "s:1418" ] }, { "annotation": null, "id": "r:1558", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 21.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1420" ] }, { "annotation": null, "id": "r:1559", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1558", "results": [ "s:1420" ] }, { "annotation": null, "id": "r:1560", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1422" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -22.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1421" ] }, { "annotation": null, "id": "r:1561", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1422" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1560", "results": [ "s:1421" ] }, { "annotation": null, "id": "r:1562", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:1422" ] }, { "annotation": null, "id": "r:1563", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:1427", "s:1430", "s:1433" ], "params": {}, "parent": null, "results": [ "s:1423" ] }, { "annotation": null, "id": "r:1564", "op": "linear.sum_nodes", "operands": [ "s:1425", "s:1428", "s:1431" ], "params": { "max_fanout": null }, "parent": "r:1563", "results": [ "s:1423" ] }, { "annotation": null, "id": "r:1565", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:1427" ], "params": {}, "parent": "r:1563", "results": [ "s:1425" ] }, { "annotation": null, "id": "r:1566", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1426" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1565", "results": [ "s:1425" ] }, { "annotation": null, "id": "r:1567", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1427" ] }, { "annotation": null, "id": "r:1568", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:1430" ], "params": {}, "parent": "r:1563", "results": [ "s:1428" ] }, { "annotation": null, "id": "r:1569", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1429" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1568", "results": [ "s:1428" ] }, { "annotation": null, "id": "r:1570", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1430" ] }, { "annotation": null, "id": "r:1571", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:1433" ], "params": {}, "parent": "r:1563", "results": [ "s:1431" ] }, { "annotation": null, "id": "r:1572", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1432" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1571", "results": [ "s:1431" ] }, { "annotation": null, "id": "r:1573", "op": "swiglu.map_select.select", "operands": [ "s:1435", "s:1368", "s:1443" ], "params": {}, "parent": null, "results": [ "s:1433" ] }, { "annotation": null, "id": "r:1574", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1434" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1573", "results": [ "s:1433" ] }, { "annotation": null, "id": "r:1575", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1436" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 5.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1435" ] }, { "annotation": null, "id": "r:1576", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1436" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1575", "results": [ "s:1435" ] }, { "annotation": null, "id": "r:1577", "op": "linear.add_const", "operands": [ "s:1437" ], "params": { "scalar": 2.0 }, "parent": null, "results": [ "s:1436" ] }, { "annotation": null, "id": "r:1578", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:1439", "s:1442" ], "params": {}, "parent": null, "results": [ "s:1437" ] }, { "annotation": null, "id": "r:1579", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1438" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1578", "results": [ "s:1437" ] }, { "annotation": null, "id": "r:1580", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1440" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1439" ] }, { "annotation": null, "id": "r:1581", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -3, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1440" ] }, { "annotation": null, "id": "r:1582", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:1442" ] }, { "annotation": null, "id": "r:1583", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1444", "s:1375", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1443" ] }, { "annotation": null, "id": "r:1584", "op": "linear.bool_to_01", "operands": [ "s:1446" ], "params": {}, "parent": null, "results": [ "s:1444" ] }, { "annotation": null, "id": "r:1585", "op": "linear.multiply_const", "operands": [ "s:1445" ], "params": { "scalar": 0.5 }, "parent": "r:1584", "results": [ "s:1444" ] }, { "annotation": null, "id": "r:1586", "op": "linear.add_const", "operands": [ "s:1446" ], "params": { "scalar": 1.0 }, "parent": "r:1584", "results": [ "s:1445" ] }, { "annotation": null, "id": "r:1587", "op": "swiglu.map_select.in_range", "operands": [ "s:1436", "s:1448" ], "params": { "n_slots": 6 }, "parent": null, "results": [ "s:1446" ] }, { "annotation": null, "id": "r:1588", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1447" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 24 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 6 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 6 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1587", "results": [ "s:1446" ] }, { "annotation": null, "id": "r:1589", "op": "linear.add_const", "operands": [ "s:1436" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1448" ] }, { "annotation": null, "id": "r:1590", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:1451", "s:1457" ], "params": {}, "parent": null, "results": [ "s:1449" ] }, { "annotation": null, "id": "r:1591", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1450" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1590", "results": [ "s:1449" ] }, { "annotation": null, "id": "r:1592", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1454", "s:1455" ], "params": {}, "parent": null, "results": [ "s:1451" ] }, { "annotation": null, "id": "r:1593", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1452" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1592", "results": [ "s:1451" ] }, { "annotation": null, "id": "r:1594", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1452" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1593", "results": [ "s:1451" ] }, { "annotation": null, "id": "r:1595", "op": "linear.sum_nodes", "operands": [ "s:1454", "s:1455" ], "params": { "max_fanout": null }, "parent": "r:1592", "results": [ "s:1452" ] }, { "annotation": null, "id": "r:1596", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 22.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1454" ] }, { "annotation": null, "id": "r:1597", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1596", "results": [ "s:1454" ] }, { "annotation": null, "id": "r:1598", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1456" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -23.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1455" ] }, { "annotation": null, "id": "r:1599", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1456" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1598", "results": [ "s:1455" ] }, { "annotation": null, "id": "r:1600", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:1456" ] }, { "annotation": null, "id": "r:1601", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:1461", "s:1464", "s:1467" ], "params": {}, "parent": null, "results": [ "s:1457" ] }, { "annotation": null, "id": "r:1602", "op": "linear.sum_nodes", "operands": [ "s:1459", "s:1462", "s:1465" ], "params": { "max_fanout": null }, "parent": "r:1601", "results": [ "s:1457" ] }, { "annotation": null, "id": "r:1603", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:1461" ], "params": {}, "parent": "r:1601", "results": [ "s:1459" ] }, { "annotation": null, "id": "r:1604", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1460" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1603", "results": [ "s:1459" ] }, { "annotation": null, "id": "r:1605", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1461" ] }, { "annotation": null, "id": "r:1606", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:1464" ], "params": {}, "parent": "r:1601", "results": [ "s:1462" ] }, { "annotation": null, "id": "r:1607", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1463" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1606", "results": [ "s:1462" ] }, { "annotation": null, "id": "r:1608", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1464" ] }, { "annotation": null, "id": "r:1609", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:1467" ], "params": {}, "parent": "r:1601", "results": [ "s:1465" ] }, { "annotation": null, "id": "r:1610", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1466" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1609", "results": [ "s:1465" ] }, { "annotation": null, "id": "r:1611", "op": "swiglu.map_select.select", "operands": [ "s:1469", "s:1368", "s:1477" ], "params": {}, "parent": null, "results": [ "s:1467" ] }, { "annotation": null, "id": "r:1612", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1468" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1611", "results": [ "s:1467" ] }, { "annotation": null, "id": "r:1613", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1470" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 5.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1469" ] }, { "annotation": null, "id": "r:1614", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1470" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1613", "results": [ "s:1469" ] }, { "annotation": null, "id": "r:1615", "op": "linear.add_const", "operands": [ "s:1471" ], "params": { "scalar": 3.0 }, "parent": null, "results": [ "s:1470" ] }, { "annotation": null, "id": "r:1616", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:1473", "s:1476" ], "params": {}, "parent": null, "results": [ "s:1471" ] }, { "annotation": null, "id": "r:1617", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1472" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1616", "results": [ "s:1471" ] }, { "annotation": null, "id": "r:1618", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1474" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1473" ] }, { "annotation": null, "id": "r:1619", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -4, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1474" ] }, { "annotation": null, "id": "r:1620", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:1476" ] }, { "annotation": null, "id": "r:1621", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1478", "s:1375", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1477" ] }, { "annotation": null, "id": "r:1622", "op": "linear.bool_to_01", "operands": [ "s:1480" ], "params": {}, "parent": null, "results": [ "s:1478" ] }, { "annotation": null, "id": "r:1623", "op": "linear.multiply_const", "operands": [ "s:1479" ], "params": { "scalar": 0.5 }, "parent": "r:1622", "results": [ "s:1478" ] }, { "annotation": null, "id": "r:1624", "op": "linear.add_const", "operands": [ "s:1480" ], "params": { "scalar": 1.0 }, "parent": "r:1622", "results": [ "s:1479" ] }, { "annotation": null, "id": "r:1625", "op": "swiglu.map_select.in_range", "operands": [ "s:1470", "s:1482" ], "params": { "n_slots": 6 }, "parent": null, "results": [ "s:1480" ] }, { "annotation": null, "id": "r:1626", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1481" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 24 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 6 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 6 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1625", "results": [ "s:1480" ] }, { "annotation": null, "id": "r:1627", "op": "linear.add_const", "operands": [ "s:1470" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1482" ] }, { "annotation": null, "id": "r:1628", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:1485", "s:1491" ], "params": {}, "parent": null, "results": [ "s:1483" ] }, { "annotation": null, "id": "r:1629", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1484" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1628", "results": [ "s:1483" ] }, { "annotation": null, "id": "r:1630", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1488", "s:1489" ], "params": {}, "parent": null, "results": [ "s:1485" ] }, { "annotation": null, "id": "r:1631", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1486" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1630", "results": [ "s:1485" ] }, { "annotation": null, "id": "r:1632", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1486" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1631", "results": [ "s:1485" ] }, { "annotation": null, "id": "r:1633", "op": "linear.sum_nodes", "operands": [ "s:1488", "s:1489" ], "params": { "max_fanout": null }, "parent": "r:1630", "results": [ "s:1486" ] }, { "annotation": null, "id": "r:1634", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 23.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1488" ] }, { "annotation": null, "id": "r:1635", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1634", "results": [ "s:1488" ] }, { "annotation": null, "id": "r:1636", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1490" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -24.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1489" ] }, { "annotation": null, "id": "r:1637", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1490" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1636", "results": [ "s:1489" ] }, { "annotation": null, "id": "r:1638", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:1490" ] }, { "annotation": null, "id": "r:1639", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:1495", "s:1498", "s:1501" ], "params": {}, "parent": null, "results": [ "s:1491" ] }, { "annotation": null, "id": "r:1640", "op": "linear.sum_nodes", "operands": [ "s:1493", "s:1496", "s:1499" ], "params": { "max_fanout": null }, "parent": "r:1639", "results": [ "s:1491" ] }, { "annotation": null, "id": "r:1641", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:1495" ], "params": {}, "parent": "r:1639", "results": [ "s:1493" ] }, { "annotation": null, "id": "r:1642", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1494" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1641", "results": [ "s:1493" ] }, { "annotation": null, "id": "r:1643", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1495" ] }, { "annotation": null, "id": "r:1644", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:1498" ], "params": {}, "parent": "r:1639", "results": [ "s:1496" ] }, { "annotation": null, "id": "r:1645", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1497" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1644", "results": [ "s:1496" ] }, { "annotation": null, "id": "r:1646", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1498" ] }, { "annotation": null, "id": "r:1647", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:1501" ], "params": {}, "parent": "r:1639", "results": [ "s:1499" ] }, { "annotation": null, "id": "r:1648", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1500" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1647", "results": [ "s:1499" ] }, { "annotation": null, "id": "r:1649", "op": "swiglu.map_select.select", "operands": [ "s:1503", "s:1368", "s:1511" ], "params": {}, "parent": null, "results": [ "s:1501" ] }, { "annotation": null, "id": "r:1650", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1502" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1649", "results": [ "s:1501" ] }, { "annotation": null, "id": "r:1651", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1504" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 5.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1503" ] }, { "annotation": null, "id": "r:1652", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1504" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1651", "results": [ "s:1503" ] }, { "annotation": null, "id": "r:1653", "op": "linear.add_const", "operands": [ "s:1505" ], "params": { "scalar": 4.0 }, "parent": null, "results": [ "s:1504" ] }, { "annotation": null, "id": "r:1654", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:1507", "s:1510" ], "params": {}, "parent": null, "results": [ "s:1505" ] }, { "annotation": null, "id": "r:1655", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1506" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1654", "results": [ "s:1505" ] }, { "annotation": null, "id": "r:1656", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1508" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1507" ] }, { "annotation": null, "id": "r:1657", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -5, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1508" ] }, { "annotation": null, "id": "r:1658", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:1510" ] }, { "annotation": null, "id": "r:1659", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1512", "s:1375", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1511" ] }, { "annotation": null, "id": "r:1660", "op": "linear.bool_to_01", "operands": [ "s:1514" ], "params": {}, "parent": null, "results": [ "s:1512" ] }, { "annotation": null, "id": "r:1661", "op": "linear.multiply_const", "operands": [ "s:1513" ], "params": { "scalar": 0.5 }, "parent": "r:1660", "results": [ "s:1512" ] }, { "annotation": null, "id": "r:1662", "op": "linear.add_const", "operands": [ "s:1514" ], "params": { "scalar": 1.0 }, "parent": "r:1660", "results": [ "s:1513" ] }, { "annotation": null, "id": "r:1663", "op": "swiglu.map_select.in_range", "operands": [ "s:1504", "s:1516" ], "params": { "n_slots": 6 }, "parent": null, "results": [ "s:1514" ] }, { "annotation": null, "id": "r:1664", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1515" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 24 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 6 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 6 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1663", "results": [ "s:1514" ] }, { "annotation": null, "id": "r:1665", "op": "linear.add_const", "operands": [ "s:1504" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1516" ] }, { "annotation": null, "id": "r:1666", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:1519", "s:1525" ], "params": {}, "parent": null, "results": [ "s:1517" ] }, { "annotation": null, "id": "r:1667", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1518" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1666", "results": [ "s:1517" ] }, { "annotation": null, "id": "r:1668", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1522", "s:1523" ], "params": {}, "parent": null, "results": [ "s:1519" ] }, { "annotation": null, "id": "r:1669", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1520" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1668", "results": [ "s:1519" ] }, { "annotation": null, "id": "r:1670", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1520" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1669", "results": [ "s:1519" ] }, { "annotation": null, "id": "r:1671", "op": "linear.sum_nodes", "operands": [ "s:1522", "s:1523" ], "params": { "max_fanout": null }, "parent": "r:1668", "results": [ "s:1520" ] }, { "annotation": null, "id": "r:1672", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 24.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1522" ] }, { "annotation": null, "id": "r:1673", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1672", "results": [ "s:1522" ] }, { "annotation": null, "id": "r:1674", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1524" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -25.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1523" ] }, { "annotation": null, "id": "r:1675", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1524" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1674", "results": [ "s:1523" ] }, { "annotation": null, "id": "r:1676", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:1524" ] }, { "annotation": null, "id": "r:1677", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:1529", "s:1532", "s:1535" ], "params": {}, "parent": null, "results": [ "s:1525" ] }, { "annotation": null, "id": "r:1678", "op": "linear.sum_nodes", "operands": [ "s:1527", "s:1530", "s:1533" ], "params": { "max_fanout": null }, "parent": "r:1677", "results": [ "s:1525" ] }, { "annotation": null, "id": "r:1679", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:1529" ], "params": {}, "parent": "r:1677", "results": [ "s:1527" ] }, { "annotation": null, "id": "r:1680", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1528" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1679", "results": [ "s:1527" ] }, { "annotation": null, "id": "r:1681", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1529" ] }, { "annotation": null, "id": "r:1682", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:1532" ], "params": {}, "parent": "r:1677", "results": [ "s:1530" ] }, { "annotation": null, "id": "r:1683", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1531" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1682", "results": [ "s:1530" ] }, { "annotation": null, "id": "r:1684", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1532" ] }, { "annotation": null, "id": "r:1685", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:1535" ], "params": {}, "parent": "r:1677", "results": [ "s:1533" ] }, { "annotation": null, "id": "r:1686", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1534" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1685", "results": [ "s:1533" ] }, { "annotation": null, "id": "r:1687", "op": "swiglu.map_select.select", "operands": [ "s:1537", "s:1368", "s:1545" ], "params": {}, "parent": null, "results": [ "s:1535" ] }, { "annotation": null, "id": "r:1688", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1536" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 66 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 66, 67 ] } } }, "parent": "r:1687", "results": [ "s:1535" ] }, { "annotation": null, "id": "r:1689", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1538" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 5.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1537" ] }, { "annotation": null, "id": "r:1690", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1538" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1689", "results": [ "s:1537" ] }, { "annotation": null, "id": "r:1691", "op": "linear.add_const", "operands": [ "s:1539" ], "params": { "scalar": 5.0 }, "parent": null, "results": [ "s:1538" ] }, { "annotation": null, "id": "r:1692", "op": "swiglu.arithmetic_ops.min", "operands": [ "s:1541", "s:1544" ], "params": {}, "parent": null, "results": [ "s:1539" ] }, { "annotation": null, "id": "r:1693", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1540" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 2 ] } }, "name": "min", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1692", "results": [ "s:1539" ] }, { "annotation": null, "id": "r:1694", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:1542" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:1541" ] }, { "annotation": null, "id": "r:1695", "op": "attention_ops.attend_to_offset", "operands": [ "s:8" ], "params": { "delta_pos": -6, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1542" ] }, { "annotation": null, "id": "r:1696", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": null, "results": [ "s:1544" ] }, { "annotation": null, "id": "r:1697", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:1546", "s:1375", "s:8" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:1545" ] }, { "annotation": null, "id": "r:1698", "op": "linear.bool_to_01", "operands": [ "s:1548" ], "params": {}, "parent": null, "results": [ "s:1546" ] }, { "annotation": null, "id": "r:1699", "op": "linear.multiply_const", "operands": [ "s:1547" ], "params": { "scalar": 0.5 }, "parent": "r:1698", "results": [ "s:1546" ] }, { "annotation": null, "id": "r:1700", "op": "linear.add_const", "operands": [ "s:1548" ], "params": { "scalar": 1.0 }, "parent": "r:1698", "results": [ "s:1547" ] }, { "annotation": null, "id": "r:1701", "op": "swiglu.map_select.in_range", "operands": [ "s:1538", "s:1550" ], "params": { "n_slots": 6 }, "parent": null, "results": [ "s:1548" ] }, { "annotation": null, "id": "r:1702", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1549" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 24 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 6 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 24, 6 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1701", "results": [ "s:1548" ] }, { "annotation": null, "id": "r:1703", "op": "linear.add_const", "operands": [ "s:1538" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:1550" ] }, { "annotation": null, "id": "r:1704", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:1553", "s:1559" ], "params": {}, "parent": null, "results": [ "s:1551" ] }, { "annotation": null, "id": "r:1705", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1552" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1704", "results": [ "s:1551" ] }, { "annotation": null, "id": "r:1706", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:1556", "s:1557" ], "params": {}, "parent": null, "results": [ "s:1553" ] }, { "annotation": null, "id": "r:1707", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1554" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:1706", "results": [ "s:1553" ] }, { "annotation": null, "id": "r:1708", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1554" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1707", "results": [ "s:1553" ] }, { "annotation": null, "id": "r:1709", "op": "linear.sum_nodes", "operands": [ "s:1556", "s:1557" ], "params": { "max_fanout": null }, "parent": "r:1706", "results": [ "s:1554" ] }, { "annotation": null, "id": "r:1710", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 25.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1556" ] }, { "annotation": null, "id": "r:1711", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1710", "results": [ "s:1556" ] }, { "annotation": null, "id": "r:1712", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:1558" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -26.5, "true_level": 1.0 }, "parent": null, "results": [ "s:1557" ] }, { "annotation": null, "id": "r:1713", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1558" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:1712", "results": [ "s:1557" ] }, { "annotation": null, "id": "r:1714", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": null, "results": [ "s:1558" ] }, { "annotation": null, "id": "r:1715", "op": "swiglu.map_select.switch", "operands": [ "s:37", "s:65", "s:74", "s:1563", "s:1566", "s:1569" ], "params": {}, "parent": null, "results": [ "s:1559" ] }, { "annotation": null, "id": "r:1716", "op": "linear.sum_nodes", "operands": [ "s:1561", "s:1564", "s:1567" ], "params": { "max_fanout": null }, "parent": "r:1715", "results": [ "s:1559" ] }, { "annotation": null, "id": "r:1717", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:37", "s:1563" ], "params": {}, "parent": "r:1715", "results": [ "s:1561" ] }, { "annotation": null, "id": "r:1718", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1562" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1717", "results": [ "s:1561" ] }, { "annotation": null, "id": "r:1719", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1563" ] }, { "annotation": null, "id": "r:1720", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:65", "s:1566" ], "params": {}, "parent": "r:1715", "results": [ "s:1564" ] }, { "annotation": null, "id": "r:1721", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1565" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1720", "results": [ "s:1564" ] }, { "annotation": null, "id": "r:1722", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1566" ] }, { "annotation": null, "id": "r:1723", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:1569" ], "params": {}, "parent": "r:1715", "results": [ "s:1567" ] }, { "annotation": null, "id": "r:1724", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1568" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 33 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 33 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 33, 34 ] } } }, "parent": "r:1723", "results": [ "s:1567" ] }, { "annotation": null, "id": "r:1725", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1569" ] }, { "annotation": null, "id": "r:1726", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 33 ] } } }, "parent": null, "results": [ "s:1570" ] } ], "sha256": "7e6b50441d050cda4dc70be518abcf6ed2a77b67aeeb1ffe7ea87a672ccd27e0" } }, "integrity": { "scope": "every manifest section except this one", "sha256": "640537b13d97d4e9f5792e6f2bc0c11a6e0931b4e226dd27fadcdb425558134b" }, "intervention_contract": { "guarantee": "The manifest describes the unmodified compiled program. Interventions outside a node's stated value contract are not covered by torchwright's semantic or numerical guarantees.", "node_value_contracts": "graphs.source.nodes[*].value_contract", "physical_coordinates": { "attention_heads": "schedule.layers[*].attention_operations[*].heads", "mlp_neurons": "schedule.layers[*].mlp_operations[*].hidden_slots", "residual": "schedule.layers[*].attention_operations[*] and mlp_operations[*] target/source/query/key column runs" } }, "metadata": { "source": { "arguments": { "max_digits": 3 }, "entrypoint": "examples.calculator_scratchpad:create_network_parts" }, "task": { "example": "calculator_scratchpad", "max_digits": 3 } }, "model": { "activation": "swish", "active_attention_heads_per_layer": [ 4, 32, 32, 22, 16, 27, 27, 11, 32, 15, 10, 9, 13, 22, 16, 11, 29, 32 ], "active_mlp_neurons_per_layer": [ 49, 156, 189, 314, 1396, 699, 1243, 1527, 459, 318, 168, 1889, 218, 987, 1221, 904, 337, 135 ], "architecture": "phi3", "bias": false, "d_head": 64, "d_model": 2048, "max_sequence_length": 512, "n_layers": 18, "rms_norm": true, "rms_norm_eps": 1e-05, "rope_base": 500000.0, "rotary_width": 64, "stored_attention_heads": 32, "stored_mlp_neurons": 1889 }, "observability": { "abstract_residual_states": [ { "hook": "blocks.0.hook_resid_pre", "state": "input" }, { "hook": "blocks.0.hook_resid_post", "state": "L0.post" }, { "hook": "blocks.1.hook_resid_post", "state": "L1.post" }, { "hook": "blocks.2.hook_resid_post", "state": "L2.post" }, { "hook": "blocks.3.hook_resid_post", "state": "L3.post" }, { "hook": "blocks.4.hook_resid_post", "state": "L4.post" }, { "hook": "blocks.5.hook_resid_post", "state": "L5.post" }, { "hook": "blocks.6.hook_resid_post", "state": "L6.post" }, { "hook": "blocks.7.hook_resid_post", "state": "L7.post" }, { "hook": "blocks.8.hook_resid_post", "state": "L8.post" }, { "hook": "blocks.9.hook_resid_post", "state": "L9.post" }, { "hook": "blocks.10.hook_resid_post", "state": "L10.post" }, { "hook": "blocks.11.hook_resid_post", "state": "L11.post" }, { "hook": "blocks.12.hook_resid_post", "state": "L12.post" }, { "hook": "blocks.13.hook_resid_post", "state": "L13.post" }, { "hook": "blocks.14.hook_resid_post", "state": "L14.post" }, { "hook": "blocks.15.hook_resid_post", "state": "L15.post" }, { "hook": "blocks.16.hook_resid_post", "state": "L16.post" }, { "hook": "blocks.17.hook_resid_post", "state": "L17.post" } ], "transformer_lens": { "attention": { "head_axis": 2, "k": "blocks.{layer}.attn.hook_k", "pattern": "blocks.{layer}.attn.hook_pattern", "q": "blocks.{layer}.attn.hook_q", "result": "blocks.{layer}.attn.hook_result", "v": "blocks.{layer}.attn.hook_v" }, "mlp": { "neuron_axis": -1, "post": "blocks.{layer}.mlp.hook_post", "pre": "blocks.{layer}.mlp.hook_pre" }, "note": "Use schedule.layers head and hidden-slot ranges to slice the standard hooks. Hook availability depends on the adapter version.", "residual_states": [ { "hook": "blocks.0.hook_resid_pre", "state": "input" }, { "hook": "blocks.0.hook_resid_post", "state": "L0.post" }, { "hook": "blocks.1.hook_resid_post", "state": "L1.post" }, { "hook": "blocks.2.hook_resid_post", "state": "L2.post" }, { "hook": "blocks.3.hook_resid_post", "state": "L3.post" }, { "hook": "blocks.4.hook_resid_post", "state": "L4.post" }, { "hook": "blocks.5.hook_resid_post", "state": "L5.post" }, { "hook": "blocks.6.hook_resid_post", "state": "L6.post" }, { "hook": "blocks.7.hook_resid_post", "state": "L7.post" }, { "hook": "blocks.8.hook_resid_post", "state": "L8.post" }, { "hook": "blocks.9.hook_resid_post", "state": "L9.post" }, { "hook": "blocks.10.hook_resid_post", "state": "L10.post" }, { "hook": "blocks.11.hook_resid_post", "state": "L11.post" }, { "hook": "blocks.12.hook_resid_post", "state": "L12.post" }, { "hook": "blocks.13.hook_resid_post", "state": "L13.post" }, { "hook": "blocks.14.hook_resid_post", "state": "L14.post" }, { "hook": "blocks.15.hook_resid_post", "state": "L15.post" }, { "hook": "blocks.16.hook_resid_post", "state": "L16.post" }, { "hook": "blocks.17.hook_resid_post", "state": "L17.post" } ] } }, "parameter_support": { "file": "torchwright_schematic_support.npz", "format": "torchwright.csr_support.v1", "sha256": "0c7a1c7db3a422246f76b1d505d9eeab25934f7fe1ff1d2b02d8ec3835fa64c8", "tensors": { "model.embed_tokens.weight": { "chunks": [ { "indices": "tensor_000072_chunk_000000_indices", "indptr": "tensor_000072_chunk_000000_indptr", "nnz": 135, "row_count": 34, "row_start": 0 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 135, "shape": [ 34, 2048 ] }, "model.layers.0.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000074_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.0.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000003_chunk_000000_indices", "indptr": "tensor_000003_chunk_000000_indptr", "nnz": 144, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000003_chunk_000001_indices", "indptr": "tensor_000003_chunk_000001_indptr", "nnz": 11, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000003_chunk_000002_indices", "indptr": "tensor_000003_chunk_000002_indptr", "nnz": 0, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000003_chunk_000003_indices", "indptr": "tensor_000003_chunk_000003_indptr", "nnz": 0, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 155, "shape": [ 2048, 1889 ] }, "model.layers.0.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000002_chunk_000000_indices", "indptr": "tensor_000002_chunk_000000_indptr", "nnz": 206, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000002_chunk_000001_indices", "indptr": "tensor_000002_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000002_chunk_000002_indices", "indptr": "tensor_000002_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000002_chunk_000003_indices", "indptr": "tensor_000002_chunk_000003_indptr", "nnz": 49, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000002_chunk_000004_indices", "indptr": "tensor_000002_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000002_chunk_000005_indices", "indptr": "tensor_000002_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000002_chunk_000006_indices", "indptr": "tensor_000002_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000002_chunk_000007_indices", "indptr": "tensor_000002_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 255, "shape": [ 3778, 2048 ] }, "model.layers.0.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000075_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.0.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000001_chunk_000000_indices", "indptr": "tensor_000001_chunk_000000_indptr", "nnz": 51, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000001_chunk_000001_indices", "indptr": "tensor_000001_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000001_chunk_000002_indices", "indptr": "tensor_000001_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000001_chunk_000003_indices", "indptr": "tensor_000001_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 51, "shape": [ 2048, 2048 ] }, "model.layers.0.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000000_chunk_000000_indices", "indptr": "tensor_000000_chunk_000000_indptr", "nnz": 256, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000000_chunk_000001_indices", "indptr": "tensor_000000_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000000_chunk_000002_indices", "indptr": "tensor_000000_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000000_chunk_000003_indices", "indptr": "tensor_000000_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000000_chunk_000004_indices", "indptr": "tensor_000000_chunk_000004_indptr", "nnz": 256, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000000_chunk_000005_indices", "indptr": "tensor_000000_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000000_chunk_000006_indices", "indptr": "tensor_000000_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000000_chunk_000007_indices", "indptr": "tensor_000000_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000000_chunk_000008_indices", "indptr": "tensor_000000_chunk_000008_indptr", "nnz": 132, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000000_chunk_000009_indices", "indptr": "tensor_000000_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000000_chunk_000010_indices", "indptr": "tensor_000000_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000000_chunk_000011_indices", "indptr": "tensor_000000_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 644, "shape": [ 6144, 2048 ] }, "model.layers.1.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000076_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.1.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000007_chunk_000000_indices", "indptr": "tensor_000007_chunk_000000_indptr", "nnz": 74, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000007_chunk_000001_indices", "indptr": "tensor_000007_chunk_000001_indptr", "nnz": 6, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000007_chunk_000002_indices", "indptr": "tensor_000007_chunk_000002_indptr", "nnz": 0, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000007_chunk_000003_indices", "indptr": "tensor_000007_chunk_000003_indptr", "nnz": 102, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 182, "shape": [ 2048, 1889 ] }, "model.layers.1.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000006_chunk_000000_indices", "indptr": "tensor_000006_chunk_000000_indptr", "nnz": 783, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000006_chunk_000001_indices", "indptr": "tensor_000006_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000006_chunk_000002_indices", "indptr": "tensor_000006_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000006_chunk_000003_indices", "indptr": "tensor_000006_chunk_000003_indptr", "nnz": 156, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000006_chunk_000004_indices", "indptr": "tensor_000006_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000006_chunk_000005_indices", "indptr": "tensor_000006_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000006_chunk_000006_indices", "indptr": "tensor_000006_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000006_chunk_000007_indices", "indptr": "tensor_000006_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 939, "shape": [ 3778, 2048 ] }, "model.layers.1.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000077_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.1.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000005_chunk_000000_indices", "indptr": "tensor_000005_chunk_000000_indptr", "nnz": 0, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000005_chunk_000001_indices", "indptr": "tensor_000005_chunk_000001_indptr", "nnz": 193, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000005_chunk_000002_indices", "indptr": "tensor_000005_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000005_chunk_000003_indices", "indptr": "tensor_000005_chunk_000003_indptr", "nnz": 255, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 960, "shape": [ 2048, 2048 ] }, "model.layers.1.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000004_chunk_000000_indices", "indptr": "tensor_000004_chunk_000000_indptr", "nnz": 406, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000004_chunk_000001_indices", "indptr": "tensor_000004_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000004_chunk_000002_indices", "indptr": "tensor_000004_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000004_chunk_000003_indices", "indptr": "tensor_000004_chunk_000003_indptr", "nnz": 459, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000004_chunk_000004_indices", "indptr": "tensor_000004_chunk_000004_indptr", "nnz": 406, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000004_chunk_000005_indices", "indptr": "tensor_000004_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000004_chunk_000006_indices", "indptr": "tensor_000004_chunk_000006_indptr", "nnz": 512, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000004_chunk_000007_indices", "indptr": "tensor_000004_chunk_000007_indptr", "nnz": 459, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000004_chunk_000008_indices", "indptr": "tensor_000004_chunk_000008_indptr", "nnz": 200, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000004_chunk_000009_indices", "indptr": "tensor_000004_chunk_000009_indptr", "nnz": 264, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000004_chunk_000010_indices", "indptr": "tensor_000004_chunk_000010_indptr", "nnz": 264, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000004_chunk_000011_indices", "indptr": "tensor_000004_chunk_000011_indptr", "nnz": 232, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 4738, "shape": [ 6144, 2048 ] }, "model.layers.10.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000094_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.10.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000043_chunk_000000_indices", "indptr": "tensor_000043_chunk_000000_indptr", "nnz": 110, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000043_chunk_000001_indices", "indptr": "tensor_000043_chunk_000001_indptr", "nnz": 58, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000043_chunk_000002_indices", "indptr": "tensor_000043_chunk_000002_indptr", "nnz": 0, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000043_chunk_000003_indices", "indptr": "tensor_000043_chunk_000003_indptr", "nnz": 0, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 168, "shape": [ 2048, 1889 ] }, "model.layers.10.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000042_chunk_000000_indices", "indptr": "tensor_000042_chunk_000000_indptr", "nnz": 436, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000042_chunk_000001_indices", "indptr": "tensor_000042_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000042_chunk_000002_indices", "indptr": "tensor_000042_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000042_chunk_000003_indices", "indptr": "tensor_000042_chunk_000003_indptr", "nnz": 159, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000042_chunk_000004_indices", "indptr": "tensor_000042_chunk_000004_indptr", "nnz": 9, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000042_chunk_000005_indices", "indptr": "tensor_000042_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000042_chunk_000006_indices", "indptr": "tensor_000042_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000042_chunk_000007_indices", "indptr": "tensor_000042_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 604, "shape": [ 3778, 2048 ] }, "model.layers.10.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000095_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.10.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000041_chunk_000000_indices", "indptr": "tensor_000041_chunk_000000_indptr", "nnz": 311, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000041_chunk_000001_indices", "indptr": "tensor_000041_chunk_000001_indptr", "nnz": 70, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000041_chunk_000002_indices", "indptr": "tensor_000041_chunk_000002_indptr", "nnz": 101, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000041_chunk_000003_indices", "indptr": "tensor_000041_chunk_000003_indptr", "nnz": 4, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 486, "shape": [ 2048, 2048 ] }, "model.layers.10.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000040_chunk_000000_indices", "indptr": "tensor_000040_chunk_000000_indptr", "nnz": 332, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000040_chunk_000001_indices", "indptr": "tensor_000040_chunk_000001_indptr", "nnz": 128, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000040_chunk_000002_indices", "indptr": "tensor_000040_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000040_chunk_000003_indices", "indptr": "tensor_000040_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000040_chunk_000004_indices", "indptr": "tensor_000040_chunk_000004_indptr", "nnz": 332, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000040_chunk_000005_indices", "indptr": "tensor_000040_chunk_000005_indptr", "nnz": 128, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000040_chunk_000006_indices", "indptr": "tensor_000040_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000040_chunk_000007_indices", "indptr": "tensor_000040_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000040_chunk_000008_indices", "indptr": "tensor_000040_chunk_000008_indptr", "nnz": 419, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000040_chunk_000009_indices", "indptr": "tensor_000040_chunk_000009_indptr", "nnz": 67, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000040_chunk_000010_indices", "indptr": "tensor_000040_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000040_chunk_000011_indices", "indptr": "tensor_000040_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1406, "shape": [ 6144, 2048 ] }, "model.layers.11.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000096_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.11.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000047_chunk_000000_indices", "indptr": "tensor_000047_chunk_000000_indptr", "nnz": 741, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000047_chunk_000001_indices", "indptr": "tensor_000047_chunk_000001_indptr", "nnz": 2270, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000047_chunk_000002_indices", "indptr": "tensor_000047_chunk_000002_indptr", "nnz": 0, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000047_chunk_000003_indices", "indptr": "tensor_000047_chunk_000003_indptr", "nnz": 0, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 3011, "shape": [ 2048, 1889 ] }, "model.layers.11.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000046_chunk_000000_indices", "indptr": "tensor_000046_chunk_000000_indptr", "nnz": 1534, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000046_chunk_000001_indices", "indptr": "tensor_000046_chunk_000001_indptr", "nnz": 1659, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000046_chunk_000002_indices", "indptr": "tensor_000046_chunk_000002_indptr", "nnz": 2048, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000046_chunk_000003_indices", "indptr": "tensor_000046_chunk_000003_indptr", "nnz": 1419, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000046_chunk_000004_indices", "indptr": "tensor_000046_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000046_chunk_000005_indices", "indptr": "tensor_000046_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000046_chunk_000006_indices", "indptr": "tensor_000046_chunk_000006_indptr", "nnz": 512, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000046_chunk_000007_indices", "indptr": "tensor_000046_chunk_000007_indptr", "nnz": 194, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 8390, "shape": [ 3778, 2048 ] }, "model.layers.11.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000097_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.11.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000045_chunk_000000_indices", "indptr": "tensor_000045_chunk_000000_indptr", "nnz": 231, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000045_chunk_000001_indices", "indptr": "tensor_000045_chunk_000001_indptr", "nnz": 68, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000045_chunk_000002_indices", "indptr": "tensor_000045_chunk_000002_indptr", "nnz": 106, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000045_chunk_000003_indices", "indptr": "tensor_000045_chunk_000003_indptr", "nnz": 36, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 441, "shape": [ 2048, 2048 ] }, "model.layers.11.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000044_chunk_000000_indices", "indptr": "tensor_000044_chunk_000000_indptr", "nnz": 353, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000044_chunk_000001_indices", "indptr": "tensor_000044_chunk_000001_indptr", "nnz": 64, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000044_chunk_000002_indices", "indptr": "tensor_000044_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000044_chunk_000003_indices", "indptr": "tensor_000044_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000044_chunk_000004_indices", "indptr": "tensor_000044_chunk_000004_indptr", "nnz": 353, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000044_chunk_000005_indices", "indptr": "tensor_000044_chunk_000005_indptr", "nnz": 64, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000044_chunk_000006_indices", "indptr": "tensor_000044_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000044_chunk_000007_indices", "indptr": "tensor_000044_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000044_chunk_000008_indices", "indptr": "tensor_000044_chunk_000008_indptr", "nnz": 419, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000044_chunk_000009_indices", "indptr": "tensor_000044_chunk_000009_indptr", "nnz": 22, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000044_chunk_000010_indices", "indptr": "tensor_000044_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000044_chunk_000011_indices", "indptr": "tensor_000044_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1275, "shape": [ 6144, 2048 ] }, "model.layers.12.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000098_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.12.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000051_chunk_000000_indices", "indptr": "tensor_000051_chunk_000000_indptr", "nnz": 170, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000051_chunk_000001_indices", "indptr": "tensor_000051_chunk_000001_indptr", "nnz": 119, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000051_chunk_000002_indices", "indptr": "tensor_000051_chunk_000002_indptr", "nnz": 0, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000051_chunk_000003_indices", "indptr": "tensor_000051_chunk_000003_indptr", "nnz": 0, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 289, "shape": [ 2048, 1889 ] }, "model.layers.12.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000050_chunk_000000_indices", "indptr": "tensor_000050_chunk_000000_indptr", "nnz": 312, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000050_chunk_000001_indices", "indptr": "tensor_000050_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000050_chunk_000002_indices", "indptr": "tensor_000050_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000050_chunk_000003_indices", "indptr": "tensor_000050_chunk_000003_indptr", "nnz": 159, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000050_chunk_000004_indices", "indptr": "tensor_000050_chunk_000004_indptr", "nnz": 59, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000050_chunk_000005_indices", "indptr": "tensor_000050_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000050_chunk_000006_indices", "indptr": "tensor_000050_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000050_chunk_000007_indices", "indptr": "tensor_000050_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 530, "shape": [ 3778, 2048 ] }, "model.layers.12.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000099_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.12.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000049_chunk_000000_indices", "indptr": "tensor_000049_chunk_000000_indptr", "nnz": 243, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000049_chunk_000001_indices", "indptr": "tensor_000049_chunk_000001_indptr", "nnz": 43, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000049_chunk_000002_indices", "indptr": "tensor_000049_chunk_000002_indptr", "nnz": 4, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000049_chunk_000003_indices", "indptr": "tensor_000049_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 290, "shape": [ 2048, 2048 ] }, "model.layers.12.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000048_chunk_000000_indices", "indptr": "tensor_000048_chunk_000000_indptr", "nnz": 512, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000048_chunk_000001_indices", "indptr": "tensor_000048_chunk_000001_indptr", "nnz": 320, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000048_chunk_000002_indices", "indptr": "tensor_000048_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000048_chunk_000003_indices", "indptr": "tensor_000048_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000048_chunk_000004_indices", "indptr": "tensor_000048_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000048_chunk_000005_indices", "indptr": "tensor_000048_chunk_000005_indptr", "nnz": 320, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000048_chunk_000006_indices", "indptr": "tensor_000048_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000048_chunk_000007_indices", "indptr": "tensor_000048_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000048_chunk_000008_indices", "indptr": "tensor_000048_chunk_000008_indptr", "nnz": 15, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000048_chunk_000009_indices", "indptr": "tensor_000048_chunk_000009_indptr", "nnz": 276, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000048_chunk_000010_indices", "indptr": "tensor_000048_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000048_chunk_000011_indices", "indptr": "tensor_000048_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1955, "shape": [ 6144, 2048 ] }, "model.layers.13.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000100_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.13.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000055_chunk_000000_indices", "indptr": "tensor_000055_chunk_000000_indptr", "nnz": 1613, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000055_chunk_000001_indices", "indptr": "tensor_000055_chunk_000001_indptr", "nnz": 326, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000055_chunk_000002_indices", "indptr": "tensor_000055_chunk_000002_indptr", "nnz": 602, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000055_chunk_000003_indices", "indptr": "tensor_000055_chunk_000003_indptr", "nnz": 0, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2541, "shape": [ 2048, 1889 ] }, "model.layers.13.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000054_chunk_000000_indices", "indptr": "tensor_000054_chunk_000000_indptr", "nnz": 1020, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000054_chunk_000001_indices", "indptr": "tensor_000054_chunk_000001_indptr", "nnz": 829, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000054_chunk_000002_indices", "indptr": "tensor_000054_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000054_chunk_000003_indices", "indptr": "tensor_000054_chunk_000003_indptr", "nnz": 159, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000054_chunk_000004_indices", "indptr": "tensor_000054_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000054_chunk_000005_indices", "indptr": "tensor_000054_chunk_000005_indptr", "nnz": 316, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000054_chunk_000006_indices", "indptr": "tensor_000054_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000054_chunk_000007_indices", "indptr": "tensor_000054_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2836, "shape": [ 3778, 2048 ] }, "model.layers.13.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000101_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.13.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000053_chunk_000000_indices", "indptr": "tensor_000053_chunk_000000_indptr", "nnz": 140, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000053_chunk_000001_indices", "indptr": "tensor_000053_chunk_000001_indptr", "nnz": 13, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000053_chunk_000002_indices", "indptr": "tensor_000053_chunk_000002_indptr", "nnz": 2, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000053_chunk_000003_indices", "indptr": "tensor_000053_chunk_000003_indptr", "nnz": 7, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 162, "shape": [ 2048, 2048 ] }, "model.layers.13.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000052_chunk_000000_indices", "indptr": "tensor_000052_chunk_000000_indptr", "nnz": 512, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000052_chunk_000001_indices", "indptr": "tensor_000052_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000052_chunk_000002_indices", "indptr": "tensor_000052_chunk_000002_indptr", "nnz": 384, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000052_chunk_000003_indices", "indptr": "tensor_000052_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000052_chunk_000004_indices", "indptr": "tensor_000052_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000052_chunk_000005_indices", "indptr": "tensor_000052_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000052_chunk_000006_indices", "indptr": "tensor_000052_chunk_000006_indptr", "nnz": 384, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000052_chunk_000007_indices", "indptr": "tensor_000052_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000052_chunk_000008_indices", "indptr": "tensor_000052_chunk_000008_indptr", "nnz": 8, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000052_chunk_000009_indices", "indptr": "tensor_000052_chunk_000009_indptr", "nnz": 12, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000052_chunk_000010_indices", "indptr": "tensor_000052_chunk_000010_indptr", "nnz": 142, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000052_chunk_000011_indices", "indptr": "tensor_000052_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2978, "shape": [ 6144, 2048 ] }, "model.layers.14.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000102_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.14.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000059_chunk_000000_indices", "indptr": "tensor_000059_chunk_000000_indptr", "nnz": 1040, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000059_chunk_000001_indices", "indptr": "tensor_000059_chunk_000001_indptr", "nnz": 27, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000059_chunk_000002_indices", "indptr": "tensor_000059_chunk_000002_indptr", "nnz": 398, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000059_chunk_000003_indices", "indptr": "tensor_000059_chunk_000003_indptr", "nnz": 337, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1802, "shape": [ 2048, 1889 ] }, "model.layers.14.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000058_chunk_000000_indices", "indptr": "tensor_000058_chunk_000000_indptr", "nnz": 902, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000058_chunk_000001_indices", "indptr": "tensor_000058_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000058_chunk_000002_indices", "indptr": "tensor_000058_chunk_000002_indptr", "nnz": 197, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000058_chunk_000003_indices", "indptr": "tensor_000058_chunk_000003_indptr", "nnz": 159, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000058_chunk_000004_indices", "indptr": "tensor_000058_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000058_chunk_000005_indices", "indptr": "tensor_000058_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000058_chunk_000006_indices", "indptr": "tensor_000058_chunk_000006_indptr", "nnz": 38, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000058_chunk_000007_indices", "indptr": "tensor_000058_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2832, "shape": [ 3778, 2048 ] }, "model.layers.14.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000103_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.14.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000057_chunk_000000_indices", "indptr": "tensor_000057_chunk_000000_indptr", "nnz": 365, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000057_chunk_000001_indices", "indptr": "tensor_000057_chunk_000001_indptr", "nnz": 83, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000057_chunk_000002_indices", "indptr": "tensor_000057_chunk_000002_indptr", "nnz": 39, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000057_chunk_000003_indices", "indptr": "tensor_000057_chunk_000003_indptr", "nnz": 40, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 527, "shape": [ 2048, 2048 ] }, "model.layers.14.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000056_chunk_000000_indices", "indptr": "tensor_000056_chunk_000000_indptr", "nnz": 512, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000056_chunk_000001_indices", "indptr": "tensor_000056_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000056_chunk_000002_indices", "indptr": "tensor_000056_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000056_chunk_000003_indices", "indptr": "tensor_000056_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000056_chunk_000004_indices", "indptr": "tensor_000056_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000056_chunk_000005_indices", "indptr": "tensor_000056_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000056_chunk_000006_indices", "indptr": "tensor_000056_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000056_chunk_000007_indices", "indptr": "tensor_000056_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000056_chunk_000008_indices", "indptr": "tensor_000056_chunk_000008_indptr", "nnz": 105, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000056_chunk_000009_indices", "indptr": "tensor_000056_chunk_000009_indptr", "nnz": 422, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000056_chunk_000010_indices", "indptr": "tensor_000056_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000056_chunk_000011_indices", "indptr": "tensor_000056_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2575, "shape": [ 6144, 2048 ] }, "model.layers.15.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000104_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.15.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000063_chunk_000000_indices", "indptr": "tensor_000063_chunk_000000_indptr", "nnz": 219, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000063_chunk_000001_indices", "indptr": "tensor_000063_chunk_000001_indptr", "nnz": 178, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000063_chunk_000002_indices", "indptr": "tensor_000063_chunk_000002_indptr", "nnz": 481, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000063_chunk_000003_indices", "indptr": "tensor_000063_chunk_000003_indptr", "nnz": 27, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 905, "shape": [ 2048, 1889 ] }, "model.layers.15.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000062_chunk_000000_indices", "indptr": "tensor_000062_chunk_000000_indptr", "nnz": 676, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000062_chunk_000001_indices", "indptr": "tensor_000062_chunk_000001_indptr", "nnz": 496, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000062_chunk_000002_indices", "indptr": "tensor_000062_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000062_chunk_000003_indices", "indptr": "tensor_000062_chunk_000003_indptr", "nnz": 159, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000062_chunk_000004_indices", "indptr": "tensor_000062_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000062_chunk_000005_indices", "indptr": "tensor_000062_chunk_000005_indptr", "nnz": 233, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000062_chunk_000006_indices", "indptr": "tensor_000062_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000062_chunk_000007_indices", "indptr": "tensor_000062_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2076, "shape": [ 3778, 2048 ] }, "model.layers.15.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000105_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.15.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000061_chunk_000000_indices", "indptr": "tensor_000061_chunk_000000_indptr", "nnz": 348, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000061_chunk_000001_indices", "indptr": "tensor_000061_chunk_000001_indptr", "nnz": 45, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000061_chunk_000002_indices", "indptr": "tensor_000061_chunk_000002_indptr", "nnz": 70, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000061_chunk_000003_indices", "indptr": "tensor_000061_chunk_000003_indptr", "nnz": 176, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 639, "shape": [ 2048, 2048 ] }, "model.layers.15.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000060_chunk_000000_indices", "indptr": "tensor_000060_chunk_000000_indptr", "nnz": 512, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000060_chunk_000001_indices", "indptr": "tensor_000060_chunk_000001_indptr", "nnz": 192, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000060_chunk_000002_indices", "indptr": "tensor_000060_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000060_chunk_000003_indices", "indptr": "tensor_000060_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000060_chunk_000004_indices", "indptr": "tensor_000060_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000060_chunk_000005_indices", "indptr": "tensor_000060_chunk_000005_indptr", "nnz": 192, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000060_chunk_000006_indices", "indptr": "tensor_000060_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000060_chunk_000007_indices", "indptr": "tensor_000060_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000060_chunk_000008_indices", "indptr": "tensor_000060_chunk_000008_indptr", "nnz": 454, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000060_chunk_000009_indices", "indptr": "tensor_000060_chunk_000009_indptr", "nnz": 185, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000060_chunk_000010_indices", "indptr": "tensor_000060_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000060_chunk_000011_indices", "indptr": "tensor_000060_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2047, "shape": [ 6144, 2048 ] }, "model.layers.16.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000106_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.16.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000067_chunk_000000_indices", "indptr": "tensor_000067_chunk_000000_indptr", "nnz": 264, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000067_chunk_000001_indices", "indptr": "tensor_000067_chunk_000001_indptr", "nnz": 4, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000067_chunk_000002_indices", "indptr": "tensor_000067_chunk_000002_indptr", "nnz": 62, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000067_chunk_000003_indices", "indptr": "tensor_000067_chunk_000003_indptr", "nnz": 6, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 336, "shape": [ 2048, 1889 ] }, "model.layers.16.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000066_chunk_000000_indices", "indptr": "tensor_000066_chunk_000000_indptr", "nnz": 336, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000066_chunk_000001_indices", "indptr": "tensor_000066_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000066_chunk_000002_indices", "indptr": "tensor_000066_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000066_chunk_000003_indices", "indptr": "tensor_000066_chunk_000003_indptr", "nnz": 158, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000066_chunk_000004_indices", "indptr": "tensor_000066_chunk_000004_indptr", "nnz": 178, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000066_chunk_000005_indices", "indptr": "tensor_000066_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000066_chunk_000006_indices", "indptr": "tensor_000066_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000066_chunk_000007_indices", "indptr": "tensor_000066_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 672, "shape": [ 3778, 2048 ] }, "model.layers.16.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000107_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.16.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000065_chunk_000000_indices", "indptr": "tensor_000065_chunk_000000_indptr", "nnz": 991, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000065_chunk_000001_indices", "indptr": "tensor_000065_chunk_000001_indptr", "nnz": 86, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000065_chunk_000002_indices", "indptr": "tensor_000065_chunk_000002_indptr", "nnz": 304, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000065_chunk_000003_indices", "indptr": "tensor_000065_chunk_000003_indptr", "nnz": 273, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1654, "shape": [ 2048, 2048 ] }, "model.layers.16.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000064_chunk_000000_indices", "indptr": "tensor_000064_chunk_000000_indptr", "nnz": 512, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000064_chunk_000001_indices", "indptr": "tensor_000064_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000064_chunk_000002_indices", "indptr": "tensor_000064_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000064_chunk_000003_indices", "indptr": "tensor_000064_chunk_000003_indptr", "nnz": 320, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000064_chunk_000004_indices", "indptr": "tensor_000064_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000064_chunk_000005_indices", "indptr": "tensor_000064_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000064_chunk_000006_indices", "indptr": "tensor_000064_chunk_000006_indptr", "nnz": 512, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000064_chunk_000007_indices", "indptr": "tensor_000064_chunk_000007_indptr", "nnz": 320, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000064_chunk_000008_indices", "indptr": "tensor_000064_chunk_000008_indptr", "nnz": 396, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000064_chunk_000009_indices", "indptr": "tensor_000064_chunk_000009_indptr", "nnz": 454, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000064_chunk_000010_indices", "indptr": "tensor_000064_chunk_000010_indptr", "nnz": 512, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000064_chunk_000011_indices", "indptr": "tensor_000064_chunk_000011_indptr", "nnz": 292, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 5366, "shape": [ 6144, 2048 ] }, "model.layers.17.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000108_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.17.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000071_chunk_000000_indices", "indptr": "tensor_000071_chunk_000000_indptr", "nnz": 67, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000071_chunk_000001_indices", "indptr": "tensor_000071_chunk_000001_indptr", "nnz": 0, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000071_chunk_000002_indices", "indptr": "tensor_000071_chunk_000002_indptr", "nnz": 0, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000071_chunk_000003_indices", "indptr": "tensor_000071_chunk_000003_indptr", "nnz": 68, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 135, "shape": [ 2048, 1889 ] }, "model.layers.17.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000070_chunk_000000_indices", "indptr": "tensor_000070_chunk_000000_indptr", "nnz": 135, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000070_chunk_000001_indices", "indptr": "tensor_000070_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000070_chunk_000002_indices", "indptr": "tensor_000070_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000070_chunk_000003_indices", "indptr": "tensor_000070_chunk_000003_indptr", "nnz": 135, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000070_chunk_000004_indices", "indptr": "tensor_000070_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000070_chunk_000005_indices", "indptr": "tensor_000070_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000070_chunk_000006_indices", "indptr": "tensor_000070_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000070_chunk_000007_indices", "indptr": "tensor_000070_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 270, "shape": [ 3778, 2048 ] }, "model.layers.17.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000109_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.17.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000069_chunk_000000_indices", "indptr": "tensor_000069_chunk_000000_indptr", "nnz": 1369, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000069_chunk_000001_indices", "indptr": "tensor_000069_chunk_000001_indptr", "nnz": 407, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000069_chunk_000002_indices", "indptr": "tensor_000069_chunk_000002_indptr", "nnz": 135, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000069_chunk_000003_indices", "indptr": "tensor_000069_chunk_000003_indptr", "nnz": 107, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2018, "shape": [ 2048, 2048 ] }, "model.layers.17.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000068_chunk_000000_indices", "indptr": "tensor_000068_chunk_000000_indptr", "nnz": 512, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000068_chunk_000001_indices", "indptr": "tensor_000068_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000068_chunk_000002_indices", "indptr": "tensor_000068_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000068_chunk_000003_indices", "indptr": "tensor_000068_chunk_000003_indptr", "nnz": 512, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000068_chunk_000004_indices", "indptr": "tensor_000068_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000068_chunk_000005_indices", "indptr": "tensor_000068_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000068_chunk_000006_indices", "indptr": "tensor_000068_chunk_000006_indptr", "nnz": 512, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000068_chunk_000007_indices", "indptr": "tensor_000068_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000068_chunk_000008_indices", "indptr": "tensor_000068_chunk_000008_indptr", "nnz": 512, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000068_chunk_000009_indices", "indptr": "tensor_000068_chunk_000009_indptr", "nnz": 507, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000068_chunk_000010_indices", "indptr": "tensor_000068_chunk_000010_indptr", "nnz": 512, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000068_chunk_000011_indices", "indptr": "tensor_000068_chunk_000011_indptr", "nnz": 487, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 6114, "shape": [ 6144, 2048 ] }, "model.layers.2.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000078_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.2.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000011_chunk_000000_indices", "indptr": "tensor_000011_chunk_000000_indptr", "nnz": 8, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000011_chunk_000001_indices", "indptr": "tensor_000011_chunk_000001_indptr", "nnz": 182, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000011_chunk_000002_indices", "indptr": "tensor_000011_chunk_000002_indptr", "nnz": 0, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000011_chunk_000003_indices", "indptr": "tensor_000011_chunk_000003_indptr", "nnz": 26, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 216, "shape": [ 2048, 1889 ] }, "model.layers.2.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000010_chunk_000000_indices", "indptr": "tensor_000010_chunk_000000_indptr", "nnz": 461, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000010_chunk_000001_indices", "indptr": "tensor_000010_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000010_chunk_000002_indices", "indptr": "tensor_000010_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000010_chunk_000003_indices", "indptr": "tensor_000010_chunk_000003_indptr", "nnz": 159, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000010_chunk_000004_indices", "indptr": "tensor_000010_chunk_000004_indptr", "nnz": 30, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000010_chunk_000005_indices", "indptr": "tensor_000010_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000010_chunk_000006_indices", "indptr": "tensor_000010_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000010_chunk_000007_indices", "indptr": "tensor_000010_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 650, "shape": [ 3778, 2048 ] }, "model.layers.2.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000079_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.2.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000009_chunk_000000_indices", "indptr": "tensor_000009_chunk_000000_indptr", "nnz": 55, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000009_chunk_000001_indices", "indptr": "tensor_000009_chunk_000001_indptr", "nnz": 350, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000009_chunk_000002_indices", "indptr": "tensor_000009_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000009_chunk_000003_indices", "indptr": "tensor_000009_chunk_000003_indptr", "nnz": 412, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1329, "shape": [ 2048, 2048 ] }, "model.layers.2.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000008_chunk_000000_indices", "indptr": "tensor_000008_chunk_000000_indptr", "nnz": 333, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000008_chunk_000001_indices", "indptr": "tensor_000008_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000008_chunk_000002_indices", "indptr": "tensor_000008_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000008_chunk_000003_indices", "indptr": "tensor_000008_chunk_000003_indptr", "nnz": 512, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000008_chunk_000004_indices", "indptr": "tensor_000008_chunk_000004_indptr", "nnz": 333, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000008_chunk_000005_indices", "indptr": "tensor_000008_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000008_chunk_000006_indices", "indptr": "tensor_000008_chunk_000006_indptr", "nnz": 512, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000008_chunk_000007_indices", "indptr": "tensor_000008_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000008_chunk_000008_indices", "indptr": "tensor_000008_chunk_000008_indptr", "nnz": 168, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000008_chunk_000009_indices", "indptr": "tensor_000008_chunk_000009_indptr", "nnz": 232, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000008_chunk_000010_indices", "indptr": "tensor_000008_chunk_000010_indptr", "nnz": 512, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000008_chunk_000011_indices", "indptr": "tensor_000008_chunk_000011_indptr", "nnz": 471, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 5121, "shape": [ 6144, 2048 ] }, "model.layers.3.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000080_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.3.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000015_chunk_000000_indices", "indptr": "tensor_000015_chunk_000000_indptr", "nnz": 19, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000015_chunk_000001_indices", "indptr": "tensor_000015_chunk_000001_indptr", "nnz": 434, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000015_chunk_000002_indices", "indptr": "tensor_000015_chunk_000002_indptr", "nnz": 18, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000015_chunk_000003_indices", "indptr": "tensor_000015_chunk_000003_indptr", "nnz": 52, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 523, "shape": [ 2048, 1889 ] }, "model.layers.3.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000014_chunk_000000_indices", "indptr": "tensor_000014_chunk_000000_indptr", "nnz": 665, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000014_chunk_000001_indices", "indptr": "tensor_000014_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000014_chunk_000002_indices", "indptr": "tensor_000014_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000014_chunk_000003_indices", "indptr": "tensor_000014_chunk_000003_indptr", "nnz": 159, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000014_chunk_000004_indices", "indptr": "tensor_000014_chunk_000004_indptr", "nnz": 155, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000014_chunk_000005_indices", "indptr": "tensor_000014_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000014_chunk_000006_indices", "indptr": "tensor_000014_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000014_chunk_000007_indices", "indptr": "tensor_000014_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 979, "shape": [ 3778, 2048 ] }, "model.layers.3.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000081_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.3.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000013_chunk_000000_indices", "indptr": "tensor_000013_chunk_000000_indptr", "nnz": 93, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000013_chunk_000001_indices", "indptr": "tensor_000013_chunk_000001_indptr", "nnz": 178, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000013_chunk_000002_indices", "indptr": "tensor_000013_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000013_chunk_000003_indices", "indptr": "tensor_000013_chunk_000003_indptr", "nnz": 238, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 509, "shape": [ 2048, 2048 ] }, "model.layers.3.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000012_chunk_000000_indices", "indptr": "tensor_000012_chunk_000000_indptr", "nnz": 512, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000012_chunk_000001_indices", "indptr": "tensor_000012_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000012_chunk_000002_indices", "indptr": "tensor_000012_chunk_000002_indptr", "nnz": 384, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000012_chunk_000003_indices", "indptr": "tensor_000012_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000012_chunk_000004_indices", "indptr": "tensor_000012_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000012_chunk_000005_indices", "indptr": "tensor_000012_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000012_chunk_000006_indices", "indptr": "tensor_000012_chunk_000006_indptr", "nnz": 384, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000012_chunk_000007_indices", "indptr": "tensor_000012_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000012_chunk_000008_indices", "indptr": "tensor_000012_chunk_000008_indptr", "nnz": 168, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000012_chunk_000009_indices", "indptr": "tensor_000012_chunk_000009_indptr", "nnz": 8, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000012_chunk_000010_indices", "indptr": "tensor_000012_chunk_000010_indptr", "nnz": 333, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000012_chunk_000011_indices", "indptr": "tensor_000012_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 3325, "shape": [ 6144, 2048 ] }, "model.layers.4.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000082_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.4.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000019_chunk_000000_indices", "indptr": "tensor_000019_chunk_000000_indptr", "nnz": 314, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000019_chunk_000001_indices", "indptr": "tensor_000019_chunk_000001_indptr", "nnz": 211, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000019_chunk_000002_indices", "indptr": "tensor_000019_chunk_000002_indptr", "nnz": 530, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000019_chunk_000003_indices", "indptr": "tensor_000019_chunk_000003_indptr", "nnz": 544, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1599, "shape": [ 2048, 1889 ] }, "model.layers.4.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000018_chunk_000000_indices", "indptr": "tensor_000018_chunk_000000_indptr", "nnz": 721, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000018_chunk_000001_indices", "indptr": "tensor_000018_chunk_000001_indptr", "nnz": 614, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000018_chunk_000002_indices", "indptr": "tensor_000018_chunk_000002_indptr", "nnz": 372, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000018_chunk_000003_indices", "indptr": "tensor_000018_chunk_000003_indptr", "nnz": 159, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000018_chunk_000004_indices", "indptr": "tensor_000018_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000018_chunk_000005_indices", "indptr": "tensor_000018_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000018_chunk_000006_indices", "indptr": "tensor_000018_chunk_000006_indptr", "nnz": 213, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000018_chunk_000007_indices", "indptr": "tensor_000018_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 3103, "shape": [ 3778, 2048 ] }, "model.layers.4.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000083_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.4.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000017_chunk_000000_indices", "indptr": "tensor_000017_chunk_000000_indptr", "nnz": 66, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000017_chunk_000001_indices", "indptr": "tensor_000017_chunk_000001_indptr", "nnz": 31, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000017_chunk_000002_indices", "indptr": "tensor_000017_chunk_000002_indptr", "nnz": 6, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000017_chunk_000003_indices", "indptr": "tensor_000017_chunk_000003_indptr", "nnz": 153, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 256, "shape": [ 2048, 2048 ] }, "model.layers.4.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000016_chunk_000000_indices", "indptr": "tensor_000016_chunk_000000_indptr", "nnz": 194, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000016_chunk_000001_indices", "indptr": "tensor_000016_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000016_chunk_000002_indices", "indptr": "tensor_000016_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000016_chunk_000003_indices", "indptr": "tensor_000016_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000016_chunk_000004_indices", "indptr": "tensor_000016_chunk_000004_indptr", "nnz": 194, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000016_chunk_000005_indices", "indptr": "tensor_000016_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000016_chunk_000006_indices", "indptr": "tensor_000016_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000016_chunk_000007_indices", "indptr": "tensor_000016_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000016_chunk_000008_indices", "indptr": "tensor_000016_chunk_000008_indptr", "nnz": 8, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000016_chunk_000009_indices", "indptr": "tensor_000016_chunk_000009_indptr", "nnz": 248, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000016_chunk_000010_indices", "indptr": "tensor_000016_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000016_chunk_000011_indices", "indptr": "tensor_000016_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1668, "shape": [ 6144, 2048 ] }, "model.layers.5.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000084_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.5.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000023_chunk_000000_indices", "indptr": "tensor_000023_chunk_000000_indptr", "nnz": 466, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000023_chunk_000001_indices", "indptr": "tensor_000023_chunk_000001_indptr", "nnz": 316, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000023_chunk_000002_indices", "indptr": "tensor_000023_chunk_000002_indptr", "nnz": 9, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000023_chunk_000003_indices", "indptr": "tensor_000023_chunk_000003_indptr", "nnz": 48, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 839, "shape": [ 2048, 1889 ] }, "model.layers.5.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000022_chunk_000000_indices", "indptr": "tensor_000022_chunk_000000_indptr", "nnz": 663, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000022_chunk_000001_indices", "indptr": "tensor_000022_chunk_000001_indptr", "nnz": 187, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000022_chunk_000002_indices", "indptr": "tensor_000022_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000022_chunk_000003_indices", "indptr": "tensor_000022_chunk_000003_indptr", "nnz": 159, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000022_chunk_000004_indices", "indptr": "tensor_000022_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000022_chunk_000005_indices", "indptr": "tensor_000022_chunk_000005_indptr", "nnz": 28, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000022_chunk_000006_indices", "indptr": "tensor_000022_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000022_chunk_000007_indices", "indptr": "tensor_000022_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1549, "shape": [ 3778, 2048 ] }, "model.layers.5.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000085_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.5.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000021_chunk_000000_indices", "indptr": "tensor_000021_chunk_000000_indptr", "nnz": 564, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000021_chunk_000001_indices", "indptr": "tensor_000021_chunk_000001_indptr", "nnz": 306, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000021_chunk_000002_indices", "indptr": "tensor_000021_chunk_000002_indptr", "nnz": 301, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000021_chunk_000003_indices", "indptr": "tensor_000021_chunk_000003_indptr", "nnz": 141, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1312, "shape": [ 2048, 2048 ] }, "model.layers.5.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000020_chunk_000000_indices", "indptr": "tensor_000020_chunk_000000_indptr", "nnz": 449, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000020_chunk_000001_indices", "indptr": "tensor_000020_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000020_chunk_000002_indices", "indptr": "tensor_000020_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000020_chunk_000003_indices", "indptr": "tensor_000020_chunk_000003_indptr", "nnz": 192, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000020_chunk_000004_indices", "indptr": "tensor_000020_chunk_000004_indptr", "nnz": 449, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000020_chunk_000005_indices", "indptr": "tensor_000020_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000020_chunk_000006_indices", "indptr": "tensor_000020_chunk_000006_indptr", "nnz": 512, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000020_chunk_000007_indices", "indptr": "tensor_000020_chunk_000007_indptr", "nnz": 192, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000020_chunk_000008_indices", "indptr": "tensor_000020_chunk_000008_indptr", "nnz": 299, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000020_chunk_000009_indices", "indptr": "tensor_000020_chunk_000009_indptr", "nnz": 362, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000020_chunk_000010_indices", "indptr": "tensor_000020_chunk_000010_indptr", "nnz": 512, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000020_chunk_000011_indices", "indptr": "tensor_000020_chunk_000011_indptr", "nnz": 139, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 4642, "shape": [ 6144, 2048 ] }, "model.layers.6.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000086_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.6.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000027_chunk_000000_indices", "indptr": "tensor_000027_chunk_000000_indptr", "nnz": 8, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000027_chunk_000001_indices", "indptr": "tensor_000027_chunk_000001_indptr", "nnz": 200, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000027_chunk_000002_indices", "indptr": "tensor_000027_chunk_000002_indptr", "nnz": 685, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000027_chunk_000003_indices", "indptr": "tensor_000027_chunk_000003_indptr", "nnz": 354, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1247, "shape": [ 2048, 1889 ] }, "model.layers.6.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000026_chunk_000000_indices", "indptr": "tensor_000026_chunk_000000_indptr", "nnz": 512, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000026_chunk_000001_indices", "indptr": "tensor_000026_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000026_chunk_000002_indices", "indptr": "tensor_000026_chunk_000002_indptr", "nnz": 221, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000026_chunk_000003_indices", "indptr": "tensor_000026_chunk_000003_indptr", "nnz": 162, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000026_chunk_000004_indices", "indptr": "tensor_000026_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000026_chunk_000005_indices", "indptr": "tensor_000026_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000026_chunk_000006_indices", "indptr": "tensor_000026_chunk_000006_indptr", "nnz": 60, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000026_chunk_000007_indices", "indptr": "tensor_000026_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2491, "shape": [ 3778, 2048 ] }, "model.layers.6.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000087_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.6.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000025_chunk_000000_indices", "indptr": "tensor_000025_chunk_000000_indptr", "nnz": 477, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000025_chunk_000001_indices", "indptr": "tensor_000025_chunk_000001_indptr", "nnz": 310, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000025_chunk_000002_indices", "indptr": "tensor_000025_chunk_000002_indptr", "nnz": 228, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000025_chunk_000003_indices", "indptr": "tensor_000025_chunk_000003_indptr", "nnz": 81, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1096, "shape": [ 2048, 2048 ] }, "model.layers.6.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000024_chunk_000000_indices", "indptr": "tensor_000024_chunk_000000_indptr", "nnz": 44, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000024_chunk_000001_indices", "indptr": "tensor_000024_chunk_000001_indptr", "nnz": 148, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000024_chunk_000002_indices", "indptr": "tensor_000024_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000024_chunk_000003_indices", "indptr": "tensor_000024_chunk_000003_indptr", "nnz": 192, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000024_chunk_000004_indices", "indptr": "tensor_000024_chunk_000004_indptr", "nnz": 44, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000024_chunk_000005_indices", "indptr": "tensor_000024_chunk_000005_indptr", "nnz": 148, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000024_chunk_000006_indices", "indptr": "tensor_000024_chunk_000006_indptr", "nnz": 512, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000024_chunk_000007_indices", "indptr": "tensor_000024_chunk_000007_indptr", "nnz": 192, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000024_chunk_000008_indices", "indptr": "tensor_000024_chunk_000008_indptr", "nnz": 264, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000024_chunk_000009_indices", "indptr": "tensor_000024_chunk_000009_indptr", "nnz": 297, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000024_chunk_000010_indices", "indptr": "tensor_000024_chunk_000010_indptr", "nnz": 388, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000024_chunk_000011_indices", "indptr": "tensor_000024_chunk_000011_indptr", "nnz": 147, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2888, "shape": [ 6144, 2048 ] }, "model.layers.7.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000088_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.7.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000031_chunk_000000_indices", "indptr": "tensor_000031_chunk_000000_indptr", "nnz": 331, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000031_chunk_000001_indices", "indptr": "tensor_000031_chunk_000001_indptr", "nnz": 314, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000031_chunk_000002_indices", "indptr": "tensor_000031_chunk_000002_indptr", "nnz": 618, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000031_chunk_000003_indices", "indptr": "tensor_000031_chunk_000003_indptr", "nnz": 319, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1582, "shape": [ 2048, 1889 ] }, "model.layers.7.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000030_chunk_000000_indices", "indptr": "tensor_000030_chunk_000000_indptr", "nnz": 551, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000030_chunk_000001_indices", "indptr": "tensor_000030_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000030_chunk_000002_indices", "indptr": "tensor_000030_chunk_000002_indptr", "nnz": 503, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000030_chunk_000003_indices", "indptr": "tensor_000030_chunk_000003_indptr", "nnz": 159, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000030_chunk_000004_indices", "indptr": "tensor_000030_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000030_chunk_000005_indices", "indptr": "tensor_000030_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000030_chunk_000006_indices", "indptr": "tensor_000030_chunk_000006_indptr", "nnz": 344, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000030_chunk_000007_indices", "indptr": "tensor_000030_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 3093, "shape": [ 3778, 2048 ] }, "model.layers.7.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000089_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.7.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000029_chunk_000000_indices", "indptr": "tensor_000029_chunk_000000_indptr", "nnz": 283, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000029_chunk_000001_indices", "indptr": "tensor_000029_chunk_000001_indptr", "nnz": 198, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000029_chunk_000002_indices", "indptr": "tensor_000029_chunk_000002_indptr", "nnz": 89, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000029_chunk_000003_indices", "indptr": "tensor_000029_chunk_000003_indptr", "nnz": 7, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 577, "shape": [ 2048, 2048 ] }, "model.layers.7.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000028_chunk_000000_indices", "indptr": "tensor_000028_chunk_000000_indptr", "nnz": 512, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000028_chunk_000001_indices", "indptr": "tensor_000028_chunk_000001_indptr", "nnz": 192, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000028_chunk_000002_indices", "indptr": "tensor_000028_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000028_chunk_000003_indices", "indptr": "tensor_000028_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000028_chunk_000004_indices", "indptr": "tensor_000028_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000028_chunk_000005_indices", "indptr": "tensor_000028_chunk_000005_indptr", "nnz": 192, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000028_chunk_000006_indices", "indptr": "tensor_000028_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000028_chunk_000007_indices", "indptr": "tensor_000028_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000028_chunk_000008_indices", "indptr": "tensor_000028_chunk_000008_indptr", "nnz": 389, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000028_chunk_000009_indices", "indptr": "tensor_000028_chunk_000009_indptr", "nnz": 188, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000028_chunk_000010_indices", "indptr": "tensor_000028_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000028_chunk_000011_indices", "indptr": "tensor_000028_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1985, "shape": [ 6144, 2048 ] }, "model.layers.8.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000090_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.8.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000035_chunk_000000_indices", "indptr": "tensor_000035_chunk_000000_indptr", "nnz": 72, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000035_chunk_000001_indices", "indptr": "tensor_000035_chunk_000001_indptr", "nnz": 273, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000035_chunk_000002_indices", "indptr": "tensor_000035_chunk_000002_indptr", "nnz": 75, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000035_chunk_000003_indices", "indptr": "tensor_000035_chunk_000003_indptr", "nnz": 68, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 488, "shape": [ 2048, 1889 ] }, "model.layers.8.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000034_chunk_000000_indices", "indptr": "tensor_000034_chunk_000000_indptr", "nnz": 615, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000034_chunk_000001_indices", "indptr": "tensor_000034_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000034_chunk_000002_indices", "indptr": "tensor_000034_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000034_chunk_000003_indices", "indptr": "tensor_000034_chunk_000003_indptr", "nnz": 162, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000034_chunk_000004_indices", "indptr": "tensor_000034_chunk_000004_indptr", "nnz": 300, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000034_chunk_000005_indices", "indptr": "tensor_000034_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000034_chunk_000006_indices", "indptr": "tensor_000034_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000034_chunk_000007_indices", "indptr": "tensor_000034_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1077, "shape": [ 3778, 2048 ] }, "model.layers.8.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000091_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.8.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000033_chunk_000000_indices", "indptr": "tensor_000033_chunk_000000_indptr", "nnz": 1159, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000033_chunk_000001_indices", "indptr": "tensor_000033_chunk_000001_indptr", "nnz": 306, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000033_chunk_000002_indices", "indptr": "tensor_000033_chunk_000002_indptr", "nnz": 85, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000033_chunk_000003_indices", "indptr": "tensor_000033_chunk_000003_indptr", "nnz": 129, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1679, "shape": [ 2048, 2048 ] }, "model.layers.8.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000032_chunk_000000_indices", "indptr": "tensor_000032_chunk_000000_indptr", "nnz": 332, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000032_chunk_000001_indices", "indptr": "tensor_000032_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000032_chunk_000002_indices", "indptr": "tensor_000032_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000032_chunk_000003_indices", "indptr": "tensor_000032_chunk_000003_indptr", "nnz": 512, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000032_chunk_000004_indices", "indptr": "tensor_000032_chunk_000004_indptr", "nnz": 332, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000032_chunk_000005_indices", "indptr": "tensor_000032_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000032_chunk_000006_indices", "indptr": "tensor_000032_chunk_000006_indptr", "nnz": 512, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000032_chunk_000007_indices", "indptr": "tensor_000032_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000032_chunk_000008_indices", "indptr": "tensor_000032_chunk_000008_indptr", "nnz": 361, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000032_chunk_000009_indices", "indptr": "tensor_000032_chunk_000009_indptr", "nnz": 396, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000032_chunk_000010_indices", "indptr": "tensor_000032_chunk_000010_indptr", "nnz": 422, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000032_chunk_000011_indices", "indptr": "tensor_000032_chunk_000011_indptr", "nnz": 500, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 5415, "shape": [ 6144, 2048 ] }, "model.layers.9.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000092_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.9.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000039_chunk_000000_indices", "indptr": "tensor_000039_chunk_000000_indptr", "nnz": 317, "row_count": 555, "row_start": 0 }, { "indices": "tensor_000039_chunk_000001_indices", "indptr": "tensor_000039_chunk_000001_indptr", "nnz": 41, "row_count": 555, "row_start": 555 }, { "indices": "tensor_000039_chunk_000002_indices", "indptr": "tensor_000039_chunk_000002_indptr", "nnz": 41, "row_count": 555, "row_start": 1110 }, { "indices": "tensor_000039_chunk_000003_indices", "indptr": "tensor_000039_chunk_000003_indptr", "nnz": 0, "row_count": 383, "row_start": 1665 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 399, "shape": [ 2048, 1889 ] }, "model.layers.9.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000038_chunk_000000_indices", "indptr": "tensor_000038_chunk_000000_indptr", "nnz": 439, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000038_chunk_000001_indices", "indptr": "tensor_000038_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000038_chunk_000002_indices", "indptr": "tensor_000038_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000038_chunk_000003_indices", "indptr": "tensor_000038_chunk_000003_indptr", "nnz": 159, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000038_chunk_000004_indices", "indptr": "tensor_000038_chunk_000004_indptr", "nnz": 159, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000038_chunk_000005_indices", "indptr": "tensor_000038_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000038_chunk_000006_indices", "indptr": "tensor_000038_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000038_chunk_000007_indices", "indptr": "tensor_000038_chunk_000007_indptr", "nnz": 0, "row_count": 194, "row_start": 3584 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 757, "shape": [ 3778, 2048 ] }, "model.layers.9.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000093_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.9.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000037_chunk_000000_indices", "indptr": "tensor_000037_chunk_000000_indptr", "nnz": 505, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000037_chunk_000001_indices", "indptr": "tensor_000037_chunk_000001_indptr", "nnz": 37, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000037_chunk_000002_indices", "indptr": "tensor_000037_chunk_000002_indptr", "nnz": 9, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000037_chunk_000003_indices", "indptr": "tensor_000037_chunk_000003_indptr", "nnz": 68, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 619, "shape": [ 2048, 2048 ] }, "model.layers.9.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000036_chunk_000000_indices", "indptr": "tensor_000036_chunk_000000_indptr", "nnz": 300, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000036_chunk_000001_indices", "indptr": "tensor_000036_chunk_000001_indptr", "nnz": 448, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000036_chunk_000002_indices", "indptr": "tensor_000036_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000036_chunk_000003_indices", "indptr": "tensor_000036_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000036_chunk_000004_indices", "indptr": "tensor_000036_chunk_000004_indptr", "nnz": 300, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000036_chunk_000005_indices", "indptr": "tensor_000036_chunk_000005_indptr", "nnz": 448, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000036_chunk_000006_indices", "indptr": "tensor_000036_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000036_chunk_000007_indices", "indptr": "tensor_000036_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000036_chunk_000008_indices", "indptr": "tensor_000036_chunk_000008_indptr", "nnz": 234, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000036_chunk_000009_indices", "indptr": "tensor_000036_chunk_000009_indptr", "nnz": 385, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000036_chunk_000010_indices", "indptr": "tensor_000036_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000036_chunk_000011_indices", "indptr": "tensor_000036_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2115, "shape": [ 6144, 2048 ] }, "model.norm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000073_indices", "nnz": 2046, "shape": [ 2048 ] } }, "zero_test": "stored fp32 value != 0.0" }, "physical_layout": { "checkpoint_parameter_map": [ { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.0.self_attn.qkv_proj.weight", "logical_matrix": "L0.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.0.self_attn.qkv_proj.weight", "logical_matrix": "L0.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.0.self_attn.qkv_proj.weight", "logical_matrix": "L0.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.0.self_attn.o_proj.weight", "logical_matrix": "L0.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 49 ], "checkpoint_tensor": "model.layers.0.mlp.gate_up_proj.weight", "logical_matrix": "L0.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 49 ], "checkpoint_tensor": "model.layers.0.mlp.gate_up_proj.weight", "logical_matrix": "L0.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 49 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.0.mlp.down_proj.weight", "logical_matrix": "L0.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.1.self_attn.qkv_proj.weight", "logical_matrix": "L1.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 2048 ], "checkpoint_tensor": "model.layers.1.self_attn.qkv_proj.weight", "logical_matrix": "L1.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 2048 ], "checkpoint_tensor": "model.layers.1.self_attn.qkv_proj.weight", "logical_matrix": "L1.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.1.self_attn.o_proj.weight", "logical_matrix": "L1.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 156 ], "checkpoint_tensor": "model.layers.1.mlp.gate_up_proj.weight", "logical_matrix": "L1.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 156 ], "checkpoint_tensor": "model.layers.1.mlp.gate_up_proj.weight", "logical_matrix": "L1.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 156 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.1.mlp.down_proj.weight", "logical_matrix": "L1.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.2.self_attn.qkv_proj.weight", "logical_matrix": "L2.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 2048 ], "checkpoint_tensor": "model.layers.2.self_attn.qkv_proj.weight", "logical_matrix": "L2.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 2048 ], "checkpoint_tensor": "model.layers.2.self_attn.qkv_proj.weight", "logical_matrix": "L2.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.2.self_attn.o_proj.weight", "logical_matrix": "L2.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 189 ], "checkpoint_tensor": "model.layers.2.mlp.gate_up_proj.weight", "logical_matrix": "L2.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 189 ], "checkpoint_tensor": "model.layers.2.mlp.gate_up_proj.weight", "logical_matrix": "L2.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 189 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.2.mlp.down_proj.weight", "logical_matrix": "L2.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1408 ], "checkpoint_tensor": "model.layers.3.self_attn.qkv_proj.weight", "logical_matrix": "L3.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 1408 ], "checkpoint_tensor": "model.layers.3.self_attn.qkv_proj.weight", "logical_matrix": "L3.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 1408 ], "checkpoint_tensor": "model.layers.3.self_attn.qkv_proj.weight", "logical_matrix": "L3.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1408 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.3.self_attn.o_proj.weight", "logical_matrix": "L3.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 314 ], "checkpoint_tensor": "model.layers.3.mlp.gate_up_proj.weight", "logical_matrix": "L3.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 314 ], "checkpoint_tensor": "model.layers.3.mlp.gate_up_proj.weight", "logical_matrix": "L3.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 314 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.3.mlp.down_proj.weight", "logical_matrix": "L3.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1024 ], "checkpoint_tensor": "model.layers.4.self_attn.qkv_proj.weight", "logical_matrix": "L4.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 1024 ], "checkpoint_tensor": "model.layers.4.self_attn.qkv_proj.weight", "logical_matrix": "L4.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 1024 ], "checkpoint_tensor": "model.layers.4.self_attn.qkv_proj.weight", "logical_matrix": "L4.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1024 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.4.self_attn.o_proj.weight", "logical_matrix": "L4.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1396 ], "checkpoint_tensor": "model.layers.4.mlp.gate_up_proj.weight", "logical_matrix": "L4.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 1396 ], "checkpoint_tensor": "model.layers.4.mlp.gate_up_proj.weight", "logical_matrix": "L4.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1396 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.4.mlp.down_proj.weight", "logical_matrix": "L4.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1728 ], "checkpoint_tensor": "model.layers.5.self_attn.qkv_proj.weight", "logical_matrix": "L5.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 1728 ], "checkpoint_tensor": "model.layers.5.self_attn.qkv_proj.weight", "logical_matrix": "L5.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 1728 ], "checkpoint_tensor": "model.layers.5.self_attn.qkv_proj.weight", "logical_matrix": "L5.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1728 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.5.self_attn.o_proj.weight", "logical_matrix": "L5.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 699 ], "checkpoint_tensor": "model.layers.5.mlp.gate_up_proj.weight", "logical_matrix": "L5.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 699 ], "checkpoint_tensor": "model.layers.5.mlp.gate_up_proj.weight", "logical_matrix": "L5.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 699 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.5.mlp.down_proj.weight", "logical_matrix": "L5.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1728 ], "checkpoint_tensor": "model.layers.6.self_attn.qkv_proj.weight", "logical_matrix": "L6.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 1728 ], "checkpoint_tensor": "model.layers.6.self_attn.qkv_proj.weight", "logical_matrix": "L6.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 1728 ], "checkpoint_tensor": "model.layers.6.self_attn.qkv_proj.weight", "logical_matrix": "L6.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1728 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.6.self_attn.o_proj.weight", "logical_matrix": "L6.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1243 ], "checkpoint_tensor": "model.layers.6.mlp.gate_up_proj.weight", "logical_matrix": "L6.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 1243 ], "checkpoint_tensor": "model.layers.6.mlp.gate_up_proj.weight", "logical_matrix": "L6.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1243 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.6.mlp.down_proj.weight", "logical_matrix": "L6.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 704 ], "checkpoint_tensor": "model.layers.7.self_attn.qkv_proj.weight", "logical_matrix": "L7.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 704 ], "checkpoint_tensor": "model.layers.7.self_attn.qkv_proj.weight", "logical_matrix": "L7.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 704 ], "checkpoint_tensor": "model.layers.7.self_attn.qkv_proj.weight", "logical_matrix": "L7.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 704 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.7.self_attn.o_proj.weight", "logical_matrix": "L7.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1527 ], "checkpoint_tensor": "model.layers.7.mlp.gate_up_proj.weight", "logical_matrix": "L7.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 1527 ], "checkpoint_tensor": "model.layers.7.mlp.gate_up_proj.weight", "logical_matrix": "L7.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1527 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.7.mlp.down_proj.weight", "logical_matrix": "L7.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.8.self_attn.qkv_proj.weight", "logical_matrix": "L8.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 2048 ], "checkpoint_tensor": "model.layers.8.self_attn.qkv_proj.weight", "logical_matrix": "L8.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 2048 ], "checkpoint_tensor": "model.layers.8.self_attn.qkv_proj.weight", "logical_matrix": "L8.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.8.self_attn.o_proj.weight", "logical_matrix": "L8.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 459 ], "checkpoint_tensor": "model.layers.8.mlp.gate_up_proj.weight", "logical_matrix": "L8.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 459 ], "checkpoint_tensor": "model.layers.8.mlp.gate_up_proj.weight", "logical_matrix": "L8.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 459 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.8.mlp.down_proj.weight", "logical_matrix": "L8.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 960 ], "checkpoint_tensor": "model.layers.9.self_attn.qkv_proj.weight", "logical_matrix": "L9.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 960 ], "checkpoint_tensor": "model.layers.9.self_attn.qkv_proj.weight", "logical_matrix": "L9.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 960 ], "checkpoint_tensor": "model.layers.9.self_attn.qkv_proj.weight", "logical_matrix": "L9.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 960 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.9.self_attn.o_proj.weight", "logical_matrix": "L9.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 318 ], "checkpoint_tensor": "model.layers.9.mlp.gate_up_proj.weight", "logical_matrix": "L9.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 318 ], "checkpoint_tensor": "model.layers.9.mlp.gate_up_proj.weight", "logical_matrix": "L9.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 318 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.9.mlp.down_proj.weight", "logical_matrix": "L9.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 640 ], "checkpoint_tensor": "model.layers.10.self_attn.qkv_proj.weight", "logical_matrix": "L10.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 640 ], "checkpoint_tensor": "model.layers.10.self_attn.qkv_proj.weight", "logical_matrix": "L10.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 640 ], "checkpoint_tensor": "model.layers.10.self_attn.qkv_proj.weight", "logical_matrix": "L10.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 640 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.10.self_attn.o_proj.weight", "logical_matrix": "L10.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 168 ], "checkpoint_tensor": "model.layers.10.mlp.gate_up_proj.weight", "logical_matrix": "L10.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 168 ], "checkpoint_tensor": "model.layers.10.mlp.gate_up_proj.weight", "logical_matrix": "L10.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 168 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.10.mlp.down_proj.weight", "logical_matrix": "L10.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 576 ], "checkpoint_tensor": "model.layers.11.self_attn.qkv_proj.weight", "logical_matrix": "L11.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 576 ], "checkpoint_tensor": "model.layers.11.self_attn.qkv_proj.weight", "logical_matrix": "L11.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 576 ], "checkpoint_tensor": "model.layers.11.self_attn.qkv_proj.weight", "logical_matrix": "L11.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 576 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.11.self_attn.o_proj.weight", "logical_matrix": "L11.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1889 ], "checkpoint_tensor": "model.layers.11.mlp.gate_up_proj.weight", "logical_matrix": "L11.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 1889 ], "checkpoint_tensor": "model.layers.11.mlp.gate_up_proj.weight", "logical_matrix": "L11.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1889 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.11.mlp.down_proj.weight", "logical_matrix": "L11.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 832 ], "checkpoint_tensor": "model.layers.12.self_attn.qkv_proj.weight", "logical_matrix": "L12.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 832 ], "checkpoint_tensor": "model.layers.12.self_attn.qkv_proj.weight", "logical_matrix": "L12.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 832 ], "checkpoint_tensor": "model.layers.12.self_attn.qkv_proj.weight", "logical_matrix": "L12.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 832 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.12.self_attn.o_proj.weight", "logical_matrix": "L12.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 218 ], "checkpoint_tensor": "model.layers.12.mlp.gate_up_proj.weight", "logical_matrix": "L12.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 218 ], "checkpoint_tensor": "model.layers.12.mlp.gate_up_proj.weight", "logical_matrix": "L12.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 218 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.12.mlp.down_proj.weight", "logical_matrix": "L12.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1408 ], "checkpoint_tensor": "model.layers.13.self_attn.qkv_proj.weight", "logical_matrix": "L13.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 1408 ], "checkpoint_tensor": "model.layers.13.self_attn.qkv_proj.weight", "logical_matrix": "L13.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 1408 ], "checkpoint_tensor": "model.layers.13.self_attn.qkv_proj.weight", "logical_matrix": "L13.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1408 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.13.self_attn.o_proj.weight", "logical_matrix": "L13.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 987 ], "checkpoint_tensor": "model.layers.13.mlp.gate_up_proj.weight", "logical_matrix": "L13.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 987 ], "checkpoint_tensor": "model.layers.13.mlp.gate_up_proj.weight", "logical_matrix": "L13.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 987 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.13.mlp.down_proj.weight", "logical_matrix": "L13.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1024 ], "checkpoint_tensor": "model.layers.14.self_attn.qkv_proj.weight", "logical_matrix": "L14.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 1024 ], "checkpoint_tensor": "model.layers.14.self_attn.qkv_proj.weight", "logical_matrix": "L14.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 1024 ], "checkpoint_tensor": "model.layers.14.self_attn.qkv_proj.weight", "logical_matrix": "L14.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1024 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.14.self_attn.o_proj.weight", "logical_matrix": "L14.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1221 ], "checkpoint_tensor": "model.layers.14.mlp.gate_up_proj.weight", "logical_matrix": "L14.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 1221 ], "checkpoint_tensor": "model.layers.14.mlp.gate_up_proj.weight", "logical_matrix": "L14.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1221 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.14.mlp.down_proj.weight", "logical_matrix": "L14.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 704 ], "checkpoint_tensor": "model.layers.15.self_attn.qkv_proj.weight", "logical_matrix": "L15.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 704 ], "checkpoint_tensor": "model.layers.15.self_attn.qkv_proj.weight", "logical_matrix": "L15.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 704 ], "checkpoint_tensor": "model.layers.15.self_attn.qkv_proj.weight", "logical_matrix": "L15.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 704 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.15.self_attn.o_proj.weight", "logical_matrix": "L15.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 904 ], "checkpoint_tensor": "model.layers.15.mlp.gate_up_proj.weight", "logical_matrix": "L15.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 904 ], "checkpoint_tensor": "model.layers.15.mlp.gate_up_proj.weight", "logical_matrix": "L15.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 904 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.15.mlp.down_proj.weight", "logical_matrix": "L15.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1856 ], "checkpoint_tensor": "model.layers.16.self_attn.qkv_proj.weight", "logical_matrix": "L16.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 1856 ], "checkpoint_tensor": "model.layers.16.self_attn.qkv_proj.weight", "logical_matrix": "L16.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 1856 ], "checkpoint_tensor": "model.layers.16.self_attn.qkv_proj.weight", "logical_matrix": "L16.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1856 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.16.self_attn.o_proj.weight", "logical_matrix": "L16.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 337 ], "checkpoint_tensor": "model.layers.16.mlp.gate_up_proj.weight", "logical_matrix": "L16.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 337 ], "checkpoint_tensor": "model.layers.16.mlp.gate_up_proj.weight", "logical_matrix": "L16.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 337 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.16.mlp.down_proj.weight", "logical_matrix": "L16.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.17.self_attn.qkv_proj.weight", "logical_matrix": "L17.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 2048 ], "checkpoint_tensor": "model.layers.17.self_attn.qkv_proj.weight", "logical_matrix": "L17.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 2048 ], "checkpoint_tensor": "model.layers.17.self_attn.qkv_proj.weight", "logical_matrix": "L17.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.17.self_attn.o_proj.weight", "logical_matrix": "L17.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 135 ], "checkpoint_tensor": "model.layers.17.mlp.gate_up_proj.weight", "logical_matrix": "L17.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 1889, 135 ], "checkpoint_tensor": "model.layers.17.mlp.gate_up_proj.weight", "logical_matrix": "L17.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 135 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.17.mlp.down_proj.weight", "logical_matrix": "L17.mlp.W_out", "transform": "transpose" }, { "checkpoint_tensor": "model.embed_tokens.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.0.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.0.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.1.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.1.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.10.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.10.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.11.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.11.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.12.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.12.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.13.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.13.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.14.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.14.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.15.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.15.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.16.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.16.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.17.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.17.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.2.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.2.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.3.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.3.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.4.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.4.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.5.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.5.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.6.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.6.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.7.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.7.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.8.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.8.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.9.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.9.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.norm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" } ], "known_null_components": { "meaning": "These components are statically absent or padding. This is not an input-specific causal-relevance claim.", "never_referenced_residual_columns": [], "padded_attention_heads": [ { "head_range": [ 4, 28 ], "layer": 0 }, { "head_range": [ 22, 10 ], "layer": 3 }, { "head_range": [ 16, 16 ], "layer": 4 }, { "head_range": [ 27, 5 ], "layer": 5 }, { "head_range": [ 27, 5 ], "layer": 6 }, { "head_range": [ 11, 21 ], "layer": 7 }, { "head_range": [ 15, 17 ], "layer": 9 }, { "head_range": [ 10, 22 ], "layer": 10 }, { "head_range": [ 9, 23 ], "layer": 11 }, { "head_range": [ 13, 19 ], "layer": 12 }, { "head_range": [ 22, 10 ], "layer": 13 }, { "head_range": [ 16, 16 ], "layer": 14 }, { "head_range": [ 11, 21 ], "layer": 15 }, { "head_range": [ 29, 3 ], "layer": 16 } ], "padded_mlp_neurons": [ { "layer": 0, "neuron_range": [ 49, 1840 ] }, { "layer": 1, "neuron_range": [ 156, 1733 ] }, { "layer": 2, "neuron_range": [ 189, 1700 ] }, { "layer": 3, "neuron_range": [ 314, 1575 ] }, { "layer": 4, "neuron_range": [ 1396, 493 ] }, { "layer": 5, "neuron_range": [ 699, 1190 ] }, { "layer": 6, "neuron_range": [ 1243, 646 ] }, { "layer": 7, "neuron_range": [ 1527, 362 ] }, { "layer": 8, "neuron_range": [ 459, 1430 ] }, { "layer": 9, "neuron_range": [ 318, 1571 ] }, { "layer": 10, "neuron_range": [ 168, 1721 ] }, { "layer": 12, "neuron_range": [ 218, 1671 ] }, { "layer": 13, "neuron_range": [ 987, 902 ] }, { "layer": 14, "neuron_range": [ 1221, 668 ] }, { "layer": 15, "neuron_range": [ 904, 985 ] }, { "layer": 16, "neuron_range": [ 337, 1552 ] }, { "layer": 17, "neuron_range": [ 135, 1754 ] } ] }, "matrices": { "L0.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L0.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L0.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L0.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L0.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 49 ] }, "L0.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 49, 2048 ] }, "L0.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 49 ] }, "L1.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L1.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 2048, 2048 ] }, "L1.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L1.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L1.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 156 ] }, "L1.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 156, 2048 ] }, "L1.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 156 ] }, "L10.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 640 ] }, "L10.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 640, 2048 ] }, "L10.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 640 ] }, "L10.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 640 ] }, "L10.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 168 ] }, "L10.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 168, 2048 ] }, "L10.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 168 ] }, "L11.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 576 ] }, "L11.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 576, 2048 ] }, "L11.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 576 ] }, "L11.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 576 ] }, "L11.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1889 ] }, "L11.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 1889, 2048 ] }, "L11.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1889 ] }, "L12.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 832 ] }, "L12.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 832, 2048 ] }, "L12.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 832 ] }, "L12.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 832 ] }, "L12.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 218 ] }, "L12.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 218, 2048 ] }, "L12.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 218 ] }, "L13.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1408 ] }, "L13.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 1408, 2048 ] }, "L13.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1408 ] }, "L13.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1408 ] }, "L13.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 987 ] }, "L13.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 987, 2048 ] }, "L13.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 987 ] }, "L14.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1024 ] }, "L14.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 1024, 2048 ] }, "L14.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1024 ] }, "L14.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1024 ] }, "L14.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1221 ] }, "L14.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 1221, 2048 ] }, "L14.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1221 ] }, "L15.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 704 ] }, "L15.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 704, 2048 ] }, "L15.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 704 ] }, "L15.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 704 ] }, "L15.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 904 ] }, "L15.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 904, 2048 ] }, "L15.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 904 ] }, "L16.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1856 ] }, "L16.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 1856, 2048 ] }, "L16.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1856 ] }, "L16.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1856 ] }, "L16.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 337 ] }, "L16.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 337, 2048 ] }, "L16.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 337 ] }, "L17.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L17.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 2048, 2048 ] }, "L17.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L17.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L17.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 135 ] }, "L17.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 135, 2048 ] }, "L17.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 135 ] }, "L2.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L2.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 2048, 2048 ] }, "L2.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L2.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L2.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 189 ] }, "L2.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 189, 2048 ] }, "L2.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 189 ] }, "L3.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1408 ] }, "L3.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 1408, 2048 ] }, "L3.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1408 ] }, "L3.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1408 ] }, "L3.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 314 ] }, "L3.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 314, 2048 ] }, "L3.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 314 ] }, "L4.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1024 ] }, "L4.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 1024, 2048 ] }, "L4.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1024 ] }, "L4.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1024 ] }, "L4.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1396 ] }, "L4.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 1396, 2048 ] }, "L4.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1396 ] }, "L5.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1728 ] }, "L5.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 1728, 2048 ] }, "L5.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1728 ] }, "L5.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1728 ] }, "L5.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 699 ] }, "L5.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 699, 2048 ] }, "L5.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 699 ] }, "L6.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1728 ] }, "L6.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 1728, 2048 ] }, "L6.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1728 ] }, "L6.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 1728 ] }, "L6.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1243 ] }, "L6.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 1243, 2048 ] }, "L6.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1243 ] }, "L7.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 704 ] }, "L7.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 704, 2048 ] }, "L7.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 704 ] }, "L7.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 704 ] }, "L7.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1527 ] }, "L7.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 1527, 2048 ] }, "L7.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1527 ] }, "L8.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L8.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 2048, 2048 ] }, "L8.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L8.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L8.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 459 ] }, "L8.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 459, 2048 ] }, "L8.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 459 ] }, "L9.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 960 ] }, "L9.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 960, 2048 ] }, "L9.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 960 ] }, "L9.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 960 ] }, "L9.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 318 ] }, "L9.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 318, 2048 ] }, "L9.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 318 ] } }, "matrix_coordinates": "logical x@W compiler coordinates", "placements": { "i:0": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L17.mlp.W_out" } ], "l:0": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 38, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 40, 2 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 43, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 48, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 61, 3 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 107, 18 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 128, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 130, 3 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 330, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 391, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1790, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1854, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1856, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1859, 17 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1877, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1889, 13 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 38, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 40, 2 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 43, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 48, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 61, 3 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 107, 18 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 128, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 130, 3 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 330, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 391, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 1790, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 1854, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 1856, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 1859, 17 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 1877, 1 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 1889, 13 ], "axis1": [ 1, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 1, 66 ], "axis1": [ 1, 33 ], "matrix": "L17.mlp.W_out" } ], "l:10": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 717, 33 ], "axis1": [ 0, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1581, 6 ], "axis1": [ 0, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1986, 25 ], "axis1": [ 0, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 272, 29 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1400, 32 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1913, 1 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 2011, 1 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 2045, 1 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 301, 6 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 308, 4 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 313, 4 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 318, 50 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 368, 64 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 432, 64 ], "axis1": [ 256, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 307, 1 ], "axis1": [ 320, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 312, 1 ], "axis1": [ 320, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 317, 1 ], "axis1": [ 320, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 496, 31 ], "axis1": [ 320, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1146, 1 ], "axis1": [ 320, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1149, 1 ], "axis1": [ 320, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1151, 1 ], "axis1": [ 320, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1302, 1 ], "axis1": [ 320, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1304, 1 ], "axis1": [ 320, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1433, 24 ], "axis1": [ 320, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1458, 1 ], "axis1": [ 320, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 527, 12 ], "axis1": [ 384, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 750, 12 ], "axis1": [ 384, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 795, 7 ], "axis1": [ 384, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1459, 28 ], "axis1": [ 384, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1569, 5 ], "axis1": [ 384, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 549, 50 ], "axis1": [ 448, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 802, 14 ], "axis1": [ 448, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 599, 64 ], "axis1": [ 512, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 663, 6 ], "axis1": [ 576, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 702, 12 ], "axis1": [ 576, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 816, 12 ], "axis1": [ 576, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 834, 1 ], "axis1": [ 576, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 2012, 33 ], "axis1": [ 576, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 202, 1 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 204, 8 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 213, 9 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 235, 2 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 714, 3 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 762, 8 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 835, 1 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 840, 1 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 845, 2 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 850, 1 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 852, 1 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 855, 1 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 860, 2 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 877, 1 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 879, 2 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 888, 8 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1021, 2 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1024, 1 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1028, 2 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1081, 1 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1179, 5 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1190, 2 ], "axis1": [ 640, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 140, 3 ], "axis1": [ 704, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 152, 6 ], "axis1": [ 704, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 165, 3 ], "axis1": [ 704, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 169, 1 ], "axis1": [ 704, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 193, 2 ], "axis1": [ 704, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 684, 18 ], "axis1": [ 704, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 770, 25 ], "axis1": [ 704, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 828, 6 ], "axis1": [ 704, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 170, 1 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 175, 2 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 242, 2 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 836, 4 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 841, 4 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 847, 3 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 851, 1 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 853, 2 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 856, 4 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 862, 6 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 929, 1 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 931, 3 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1192, 31 ], "axis1": [ 768, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 896, 33 ], "axis1": [ 832, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 934, 25 ], "axis1": [ 832, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 999, 1 ], "axis1": [ 832, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 38, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 40, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 48, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 61, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 107, 18 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 128, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 130, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 330, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 391, 1 ], "matrix": "L17.attn.W_O" } ], "l:100": [ { "axis0": [ 0, 1 ], "axis1": [ 1843, 1 ], "matrix": "L3.mlp.W_out" } ], "l:1000": [ { "axis0": [ 0, 1 ], "axis1": [ 502, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 502, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 531, 33 ], "axis1": [ 502, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 502, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 531, 33 ], "axis1": [ 502, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 502, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 502, 33 ], "axis1": [ 1005, 15 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 502, 33 ], "axis1": [ 1021, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 502, 33 ], "axis1": [ 1024, 16 ], "matrix": "L7.mlp.W_out" } ], "l:1002": [ { "axis0": [ 0, 1 ], "axis1": [ 531, 33 ], "matrix": "L0.mlp.W_out" } ], "l:1003": [ { "axis0": [ 0, 1 ], "axis1": [ 535, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 535, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 762, 33 ], "axis1": [ 535, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 535, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 762, 33 ], "axis1": [ 535, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 535, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 535, 33 ], "axis1": [ 1040, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 535, 33 ], "axis1": [ 1042, 4 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 535, 33 ], "axis1": [ 1047, 4 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 535, 33 ], "axis1": [ 1052, 9 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 535, 33 ], "axis1": [ 1062, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 535, 33 ], "axis1": [ 1064, 14 ], "matrix": "L7.mlp.W_out" } ], "l:1005": [ { "axis0": [ 0, 1 ], "axis1": [ 762, 33 ], "matrix": "L0.mlp.W_out" } ], "l:1006": [ { "axis0": [ 0, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1791, 9 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1805, 3 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1809, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1811, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1813, 9 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1823, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1832, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1835, 3 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1840, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1843, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1976, 3 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1791, 9 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1805, 3 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1809, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1811, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1813, 9 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1823, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1832, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1835, 3 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1840, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1843, 1 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1976, 3 ], "axis1": [ 568, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 568, 33 ], "axis1": [ 1078, 3 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 568, 33 ], "axis1": [ 1082, 29 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 568, 33 ], "axis1": [ 1112, 1 ], "matrix": "L7.mlp.W_out" } ], "l:1008": [ { "axis0": [ 0, 1 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 233, 33 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 884, 1 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1067, 1 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1069, 4 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1078, 1 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1085, 26 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1112, 1 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 233, 33 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 884, 1 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1067, 1 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1069, 4 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1078, 1 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1085, 26 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1112, 1 ], "axis1": [ 862, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 862, 66 ], "axis1": [ 1791, 9 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 862, 66 ], "axis1": [ 1805, 3 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 862, 66 ], "axis1": [ 1809, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 862, 66 ], "axis1": [ 1811, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 862, 66 ], "axis1": [ 1813, 9 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 862, 66 ], "axis1": [ 1823, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 862, 66 ], "axis1": [ 1832, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 862, 66 ], "axis1": [ 1835, 3 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 862, 66 ], "axis1": [ 1840, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 862, 66 ], "axis1": [ 1843, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 862, 66 ], "axis1": [ 1976, 3 ], "matrix": "L6.mlp.W_out" } ], "l:1010": [ { "axis0": [ 0, 1 ], "axis1": [ 105, 5 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 105, 5 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 884, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 1015, 1 ], "axis1": [ 105, 5 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 105, 5 ], "axis1": [ 884, 1 ], "matrix": "L3.mlp.W_out" } ], "l:1011": [ { "axis0": [ 0, 1 ], "axis1": [ 151, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 156, 5 ], "axis1": [ 151, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 828, 20 ], "axis1": [ 151, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1660, 2 ], "axis1": [ 151, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1784, 6 ], "axis1": [ 151, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 156, 5 ], "axis1": [ 152, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 828, 20 ], "axis1": [ 152, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1660, 2 ], "axis1": [ 152, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1784, 6 ], "axis1": [ 152, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1015, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 152, 1 ], "axis1": [ 1015, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" } ], "l:1012": [ { "axis0": [ 149, 1 ], "axis1": [ 512, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 149, 1 ], "axis1": [ 512, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 512, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 156, 5 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 828, 20 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1660, 2 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1784, 6 ], "matrix": "L2.attn.W_O" } ], "l:1013": [ { "axis0": [ 0, 1 ], "axis1": [ 149, 1 ], "matrix": "L0.mlp.W_out" } ], "l:1014": [ { "axis0": [ 853, 2 ], "axis1": [ 256, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 856, 3 ], "axis1": [ 256, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 1798, 1 ], "axis1": [ 256, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 929, 6 ], "axis1": [ 256, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 256, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 233, 33 ], "matrix": "L6.attn.W_O" } ], "l:1015": [ { "axis0": [ 0, 1 ], "axis1": [ 13, 8 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 13, 8 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 853, 2 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 856, 3 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1798, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 1015, 1 ], "axis1": [ 13, 8 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 13, 8 ], "axis1": [ 853, 2 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 13, 8 ], "axis1": [ 856, 3 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 13, 8 ], "axis1": [ 1798, 1 ], "matrix": "L3.mlp.W_out" } ], "l:1016": [ { "axis0": [ 0, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 118, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 513, 3 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 869, 8 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 878, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 881, 7 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 930, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 966, 6 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 975, 4 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1008, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1012, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1386, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 118, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 513, 3 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 869, 8 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 878, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 881, 7 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 930, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 966, 6 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 975, 4 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 1008, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 1012, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 1386, 1 ], "axis1": [ 1834, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 1834, 33 ], "axis1": [ 170, 1 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 1834, 33 ], "axis1": [ 175, 2 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 1834, 33 ], "axis1": [ 929, 1 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 1834, 33 ], "axis1": [ 931, 28 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 1834, 33 ], "axis1": [ 999, 1 ], "matrix": "L11.mlp.W_out" } ], "l:1018": [ { "axis0": [ 0, 1 ], "axis1": [ 920, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 920, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1386, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1064, 2 ], "axis1": [ 920, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 920, 2 ], "axis1": [ 1386, 1 ], "matrix": "L4.mlp.W_out" } ], "l:102": [ { "axis0": [ 0, 1 ], "axis1": [ 81, 1 ], "matrix": "L0.mlp.W_out" } ], "l:1020": [ { "axis0": [ 0, 1 ], "axis1": [ 264, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 264, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 264, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1064, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 264, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 264, 2 ], "axis1": [ 1064, 1 ], "matrix": "L3.mlp.W_out" } ], "l:1021": [ { "axis0": [ 0, 1 ], "axis1": [ 266, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 266, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 266, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1065, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 266, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 266, 2 ], "axis1": [ 1065, 1 ], "matrix": "L3.mlp.W_out" } ], "l:1022": [ { "axis0": [ 0, 1 ], "axis1": [ 102, 66 ], "matrix": "L10.mlp.W_up" }, { "axis0": [ 140, 3 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 152, 6 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 165, 3 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 169, 2 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 175, 2 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 185, 1 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 192, 3 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 196, 1 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 198, 4 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 203, 1 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 699, 3 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 1018, 2 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 1113, 66 ], "axis1": [ 102, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 140, 3 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 152, 6 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 165, 3 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 169, 2 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 175, 2 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 185, 1 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 192, 3 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 196, 1 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 198, 4 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 203, 1 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 699, 3 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 1018, 2 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 1113, 66 ], "axis1": [ 135, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 102, 1 ], "axis1": [ 118, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 103, 3 ], "axis1": [ 513, 3 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 106, 8 ], "axis1": [ 869, 8 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 114, 1 ], "axis1": [ 878, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 115, 7 ], "axis1": [ 881, 7 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 122, 1 ], "axis1": [ 930, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 123, 6 ], "axis1": [ 966, 6 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 129, 4 ], "axis1": [ 975, 4 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 133, 1 ], "axis1": [ 1008, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 134, 1 ], "axis1": [ 1012, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 135, 1 ], "axis1": [ 118, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 136, 3 ], "axis1": [ 513, 3 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 139, 8 ], "axis1": [ 869, 8 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 147, 1 ], "axis1": [ 878, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 148, 7 ], "axis1": [ 881, 7 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 155, 1 ], "axis1": [ 930, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 156, 6 ], "axis1": [ 966, 6 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 162, 4 ], "axis1": [ 975, 4 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 166, 1 ], "axis1": [ 1008, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 167, 1 ], "axis1": [ 1012, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" } ], "l:1024": [ { "axis0": [ 0, 1 ], "axis1": [ 601, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 601, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 564, 33 ], "axis1": [ 601, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 601, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 564, 33 ], "axis1": [ 601, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 601, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 601, 33 ], "axis1": [ 1113, 33 ], "matrix": "L7.mlp.W_out" } ], "l:1026": [ { "axis0": [ 0, 1 ], "axis1": [ 564, 33 ], "matrix": "L0.mlp.W_out" } ], "l:1027": [ { "axis0": [ 0, 1 ], "axis1": [ 209, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 209, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1068, 13 ], "axis1": [ 209, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1082, 20 ], "axis1": [ 209, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 209, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1068, 13 ], "axis1": [ 209, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1082, 20 ], "axis1": [ 209, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 209, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 209, 33 ], "axis1": [ 140, 3 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 209, 33 ], "axis1": [ 152, 6 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 209, 33 ], "axis1": [ 165, 3 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 209, 33 ], "axis1": [ 169, 2 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 209, 33 ], "axis1": [ 173, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 209, 33 ], "axis1": [ 175, 2 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 209, 33 ], "axis1": [ 185, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 209, 33 ], "axis1": [ 192, 3 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 209, 33 ], "axis1": [ 196, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 209, 33 ], "axis1": [ 198, 4 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 209, 33 ], "axis1": [ 203, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 209, 33 ], "axis1": [ 331, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 209, 33 ], "axis1": [ 699, 3 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 209, 33 ], "axis1": [ 1018, 2 ], "matrix": "L9.mlp.W_out" } ], "l:1029": [ { "axis0": [ 0, 1 ], "axis1": [ 1068, 13 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1082, 20 ], "matrix": "L8.mlp.W_out" } ], "l:103": [ { "axis0": [ 0, 1 ], "axis1": [ 272, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 985, 1 ], "axis1": [ 272, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 985, 1 ], "axis1": [ 273, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 272, 1 ], "axis1": [ 1068, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 273, 1 ], "axis1": [ 1068, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1068, 1 ], "matrix": "L3.mlp.W_out" } ], "l:1030": [ { "axis0": [ 0, 1 ], "axis1": [ 634, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 634, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 634, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1979, 33 ], "axis1": [ 634, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 634, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1979, 33 ], "axis1": [ 634, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 634, 33 ], "axis1": [ 1146, 33 ], "matrix": "L7.mlp.W_out" } ], "l:1032": [ { "axis0": [ 0, 1 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 266, 33 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 883, 1 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1067, 1 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1069, 4 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1078, 1 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1085, 26 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1112, 1 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 266, 33 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 883, 1 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1067, 1 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1069, 4 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1078, 1 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1085, 26 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1112, 1 ], "axis1": [ 928, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 928, 66 ], "axis1": [ 1979, 33 ], "matrix": "L6.mlp.W_out" } ], "l:1034": [ { "axis0": [ 0, 1 ], "axis1": [ 100, 5 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 100, 5 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 883, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 1016, 1 ], "axis1": [ 100, 5 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 100, 5 ], "axis1": [ 883, 1 ], "matrix": "L3.mlp.W_out" } ], "l:1035": [ { "axis0": [ 0, 1 ], "axis1": [ 153, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 34, 1 ], "axis1": [ 153, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 848, 13 ], "axis1": [ 153, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 862, 19 ], "axis1": [ 153, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 34, 1 ], "axis1": [ 154, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 848, 13 ], "axis1": [ 154, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 862, 19 ], "axis1": [ 154, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 153, 1 ], "axis1": [ 1016, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 154, 1 ], "axis1": [ 1016, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" } ], "l:1036": [ { "axis0": [ 150, 1 ], "axis1": [ 576, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 150, 1 ], "axis1": [ 576, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 576, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 34, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 848, 13 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 862, 19 ], "matrix": "L2.attn.W_O" } ], "l:1037": [ { "axis0": [ 0, 1 ], "axis1": [ 150, 1 ], "matrix": "L0.mlp.W_out" } ], "l:1038": [ { "axis0": [ 847, 3 ], "axis1": [ 320, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 851, 1 ], "axis1": [ 320, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 1662, 1 ], "axis1": [ 320, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 1797, 1 ], "axis1": [ 320, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 929, 6 ], "axis1": [ 320, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 320, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 266, 33 ], "matrix": "L6.attn.W_O" } ], "l:1039": [ { "axis0": [ 0, 1 ], "axis1": [ 9, 4 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 4 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 847, 3 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 851, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1662, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1797, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 1016, 1 ], "axis1": [ 9, 4 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 9, 4 ], "axis1": [ 847, 3 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 9, 4 ], "axis1": [ 851, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 9, 4 ], "axis1": [ 1662, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 9, 4 ], "axis1": [ 1797, 1 ], "matrix": "L3.mlp.W_out" } ], "l:104": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 32 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 32 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 985, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1663, 1 ], "axis1": [ 5, 32 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 5, 32 ], "axis1": [ 985, 1 ], "matrix": "L2.mlp.W_out" } ], "l:1040": [ { "axis0": [ 0, 1 ], "axis1": [ 516, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 516, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 279, 33 ], "axis1": [ 516, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1389, 1 ], "axis1": [ 516, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 279, 33 ], "axis1": [ 516, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1389, 1 ], "axis1": [ 516, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 516, 33 ], "axis1": [ 896, 33 ], "matrix": "L5.mlp.W_out" } ], "l:1042": [ { "axis0": [ 0, 1 ], "axis1": [ 922, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 922, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1389, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1067, 1 ], "axis1": [ 922, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1840, 1 ], "axis1": [ 922, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 922, 2 ], "axis1": [ 1389, 1 ], "matrix": "L4.mlp.W_out" } ], "l:1044": [ { "axis0": [ 0, 1 ], "axis1": [ 268, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 268, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 268, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1840, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 268, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 268, 2 ], "axis1": [ 1840, 1 ], "matrix": "L3.mlp.W_out" } ], "l:1045": [ { "axis0": [ 0, 1 ], "axis1": [ 270, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 270, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 270, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1067, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 270, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 270, 2 ], "axis1": [ 1067, 1 ], "matrix": "L3.mlp.W_out" } ], "l:1046": [ { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 1976, 64 ], "axis1": [ 512, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 279, 33 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 834, 2 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 840, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 845, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 850, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 852, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 855, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 860, 2 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 877, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 879, 2 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 889, 2 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 894, 2 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 910, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 920, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 930, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 935, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 940, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 945, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 950, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 955, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 960, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 965, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 970, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 980, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1014, 1 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 2040, 6 ], "axis1": [ 576, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 279, 33 ], "matrix": "L5.attn.W_O" } ], "l:1048": [ { "axis0": [ 0, 1 ], "axis1": [ 775, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 775, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 597, 33 ], "axis1": [ 775, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 775, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 597, 33 ], "axis1": [ 775, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 775, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 775, 33 ], "axis1": [ 1976, 33 ], "matrix": "L4.mlp.W_out" } ], "l:105": [ { "axis0": [ 66, 1 ], "axis1": [ 64, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 831, 1 ], "axis1": [ 64, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1831, 1 ], "axis1": [ 64, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 1663, 1 ], "matrix": "L2.attn.W_O" } ], "l:1050": [ { "axis0": [ 0, 1 ], "axis1": [ 597, 33 ], "matrix": "L0.mlp.W_out" } ], "l:1051": [ { "axis0": [ 0, 1 ], "axis1": [ 808, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 808, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 795, 33 ], "axis1": [ 808, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 808, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 795, 33 ], "axis1": [ 808, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 808, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 808, 33 ], "axis1": [ 2009, 33 ], "matrix": "L4.mlp.W_out" } ], "l:1053": [ { "axis0": [ 0, 1 ], "axis1": [ 795, 33 ], "matrix": "L0.mlp.W_out" } ], "l:1054": [ { "axis0": [ 0, 1 ], "axis1": [ 841, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 841, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 366, 33 ], "axis1": [ 841, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 841, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 366, 33 ], "axis1": [ 841, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 841, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 841, 33 ], "axis1": [ 834, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 840, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 845, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 850, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 852, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 855, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 860, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 877, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 879, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 889, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 894, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 910, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 920, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 930, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 935, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 940, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 945, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 950, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 955, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 960, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 965, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 970, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 980, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 1014, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 841, 33 ], "axis1": [ 2042, 4 ], "matrix": "L4.mlp.W_out" } ], "l:1056": [ { "axis0": [ 0, 1 ], "axis1": [ 366, 33 ], "matrix": "L0.mlp.W_out" } ], "l:1057": [ { "axis0": [ 0, 1 ], "axis1": [ 1854, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1856, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1859, 17 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1877, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1889, 13 ], "matrix": "L15.mlp.W_out" } ], "l:106": [ { "axis0": [ 0, 1 ], "axis1": [ 66, 1 ], "matrix": "L0.mlp.W_out" } ], "l:107": [ { "axis0": [ 61, 2 ], "axis1": [ 0, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 42, 1 ], "axis1": [ 0, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 63, 1 ], "axis1": [ 0, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 42, 1 ], "axis1": [ 0, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 831, 1 ], "matrix": "L1.attn.W_O" } ], "l:109": [ { "axis0": [ 0, 1 ], "axis1": [ 61, 1 ], "matrix": "L0.mlp.W_out" } ], "l:110": [ { "axis0": [ 0, 1 ], "axis1": [ 62, 1 ], "matrix": "L0.mlp.W_out" } ], "l:112": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 42, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 33 ], "axis1": [ 5, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 5, 1 ], "axis1": [ 42, 1 ], "matrix": "L0.mlp.W_out" } ], "l:113": [ { "axis0": [ 0, 1 ], "axis1": [ 63, 1 ], "matrix": "L0.mlp.W_out" } ], "l:114": [ { "axis0": [ 0, 1 ], "axis1": [ 68, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 68, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 69, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 1831, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 69, 1 ], "axis1": [ 1831, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1831, 1 ], "matrix": "L1.mlp.W_out" } ], "l:115": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 84, 1 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1187, 3 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 1800, 4 ], "matrix": "L7.attn.W_O" } ], "l:117": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 323, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1184, 3 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 323, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1184, 3 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1, 3 ], "axis1": [ 1187, 3 ], "matrix": "L6.mlp.W_out" } ], "l:119": [ { "axis0": [ 0, 1 ], "axis1": [ 45, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 45, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 323, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 43, 1 ], "axis1": [ 45, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1689, 1 ], "axis1": [ 45, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 45, 2 ], "axis1": [ 323, 1 ], "matrix": "L5.mlp.W_out" } ], "l:12": [ { "axis0": [ 0, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 37, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 44, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 48, 2 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 51, 4 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 61, 2 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 72, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 75, 6 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 168, 12 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1021, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 37, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 44, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 48, 2 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 51, 4 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 58, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 61, 2 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 67, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 72, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 75, 6 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 168, 12 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1021, 1 ], "axis1": [ 417, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 417, 33 ], "axis1": [ 717, 33 ], "matrix": "L5.mlp.W_out" } ], "l:121": [ { "axis0": [ 0, 1 ], "axis1": [ 6, 10 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 6, 10 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 43, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 33 ], "axis1": [ 6, 10 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 6, 10 ], "axis1": [ 43, 1 ], "matrix": "L0.mlp.W_out" } ], "l:122": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1689, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 48, 1 ], "axis1": [ 1, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 1689, 1 ], "matrix": "L4.mlp.W_out" } ], "l:123": [ { "axis0": [ 67, 2 ], "axis1": [ 192, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 69, 1 ], "axis1": [ 192, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1796, 1 ], "axis1": [ 192, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1796, 1 ], "axis1": [ 192, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 48, 1 ], "matrix": "L4.attn.W_O" } ], "l:125": [ { "axis0": [ 0, 1 ], "axis1": [ 67, 1 ], "matrix": "L0.mlp.W_out" } ], "l:126": [ { "axis0": [ 0, 1 ], "axis1": [ 68, 1 ], "matrix": "L0.mlp.W_out" } ], "l:128": [ { "axis0": [ 0, 1 ], "axis1": [ 69, 1 ], "matrix": "L0.mlp.W_out" } ], "l:129": [ { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 320, 3 ], "axis1": [ 1088, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1088, 64 ], "axis1": [ 1184, 3 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1184, 3 ], "matrix": "L6.mlp.W_out" } ], "l:130": [ { "axis0": [ 0, 1 ], "axis1": [ 33, 12 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 33, 12 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 3 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 1139, 1 ], "axis1": [ 33, 12 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1680, 1 ], "axis1": [ 33, 12 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 33, 12 ], "axis1": [ 320, 3 ], "matrix": "L5.mlp.W_out" } ], "l:132": [ { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1068, 1 ], "axis1": [ 512, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 1680, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1680, 1 ], "matrix": "L4.mlp.W_out" } ], "l:133": [ { "axis0": [ 0, 1 ], "axis1": [ 926, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1680, 1 ], "axis1": [ 926, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1680, 1 ], "axis1": [ 927, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 926, 1 ], "axis1": [ 1139, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 927, 1 ], "axis1": [ 1139, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 926, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 927, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1139, 1 ], "matrix": "L4.mlp.W_out" } ], "l:135": [ { "axis0": [ 0, 1 ], "axis1": [ 10, 4 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 10, 4 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 10, 4 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 73, 4 ], "axis1": [ 10, 4 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 10, 4 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 73, 4 ], "axis1": [ 10, 4 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 10, 4 ], "axis1": [ 1800, 4 ], "matrix": "L1.mlp.W_out" } ], "l:137": [ { "axis0": [ 0, 1 ], "axis1": [ 73, 4 ], "matrix": "L0.mlp.W_out" } ], "l:138": [ { "axis0": [ 0, 1 ], "axis1": [ 4, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 4, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 4, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 85, 33 ], "axis1": [ 4, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 323, 1 ], "axis1": [ 4, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1, 33 ], "axis1": [ 4, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 85, 33 ], "axis1": [ 4, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 323, 1 ], "axis1": [ 4, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 4, 66 ], "axis1": [ 1190, 33 ], "matrix": "L6.mlp.W_out" } ], "l:14": [ { "axis0": [ 0, 1 ], "axis1": [ 874, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 874, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1021, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1004, 2 ], "axis1": [ 874, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 874, 2 ], "axis1": [ 1021, 1 ], "matrix": "L4.mlp.W_out" } ], "l:140": [ { "axis0": [ 0, 1 ], "axis1": [ 85, 33 ], "matrix": "L0.mlp.W_out" } ], "l:141": [ { "axis0": [ 0, 1 ], "axis1": [ 1879, 2 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 64, 1 ], "axis1": [ 1879, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 300, 21 ], "axis1": [ 1879, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 322, 1 ], "axis1": [ 1879, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 1879, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 1879, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 332, 3 ], "axis1": [ 1879, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 336, 3 ], "axis1": [ 1879, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 64, 1 ], "axis1": [ 1880, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 300, 21 ], "axis1": [ 1880, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 322, 1 ], "axis1": [ 1880, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 1880, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 1880, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 332, 3 ], "axis1": [ 1880, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 336, 3 ], "axis1": [ 1880, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1879, 1 ], "axis1": [ 192, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1880, 1 ], "axis1": [ 192, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" } ], "l:142": [ { "axis0": [ 1025, 2 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 41, 1 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 1027, 1 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 304, 23 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 332, 3 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 336, 7 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 64, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 300, 21 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 322, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 324, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 328, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 332, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 336, 3 ], "matrix": "L11.attn.W_O" } ], "l:144": [ { "axis0": [ 0, 1 ], "axis1": [ 1025, 1 ], "matrix": "L8.mlp.W_out" } ], "l:145": [ { "axis0": [ 0, 1 ], "axis1": [ 1026, 1 ], "matrix": "L8.mlp.W_out" } ], "l:147": [ { "axis0": [ 0, 1 ], "axis1": [ 1027, 1 ], "matrix": "L8.mlp.W_out" } ], "l:148": [ { "axis0": [ 339, 4 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 64, 1 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 1691, 3 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 296, 33 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 304, 23 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 332, 3 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 336, 7 ], "matrix": "L10.attn.W_O" } ], "l:149": [ { "axis0": [ 0, 1 ], "axis1": [ 89, 17 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 89, 17 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 339, 4 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 34, 1 ], "axis1": [ 89, 17 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 89, 17 ], "axis1": [ 339, 4 ], "matrix": "L9.mlp.W_out" } ], "l:150": [ { "axis0": [ 1179, 2 ], "axis1": [ 64, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 41, 1 ], "axis1": [ 64, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 82, 1 ], "axis1": [ 64, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 1183, 1 ], "axis1": [ 64, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 34, 1 ], "matrix": "L9.attn.W_O" } ], "l:152": [ { "axis0": [ 0, 1 ], "axis1": [ 1179, 1 ], "matrix": "L7.mlp.W_out" } ], "l:153": [ { "axis0": [ 0, 1 ], "axis1": [ 1180, 1 ], "matrix": "L7.mlp.W_out" } ], "l:155": [ { "axis0": [ 0, 1 ], "axis1": [ 82, 1 ], "matrix": "L0.mlp.W_out" } ], "l:156": [ { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 319, 1 ], "axis1": [ 1024, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1024, 64 ], "axis1": [ 1183, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1183, 1 ], "matrix": "L6.mlp.W_out" } ], "l:157": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 32 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 32 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 319, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 35, 1 ], "axis1": [ 1, 32 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1, 32 ], "axis1": [ 319, 1 ], "matrix": "L5.mlp.W_out" } ], "l:158": [ { "axis0": [ 1407, 1 ], "axis1": [ 64, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 1676, 1 ], "axis1": [ 64, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 1264, 1 ], "axis1": [ 64, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 35, 1 ], "matrix": "L5.attn.W_O" } ], "l:159": [ { "axis0": [ 0, 1 ], "axis1": [ 1407, 1 ], "matrix": "L4.mlp.W_out" } ], "l:16": [ { "axis0": [ 0, 1 ], "axis1": [ 164, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 164, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 164, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1004, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 164, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 164, 2 ], "axis1": [ 1004, 1 ], "matrix": "L3.mlp.W_out" } ], "l:160": [ { "axis0": [ 64, 2 ], "axis1": [ 128, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 1079, 1 ], "axis1": [ 128, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1796, 1 ], "axis1": [ 128, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1796, 1 ], "axis1": [ 128, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 1676, 1 ], "matrix": "L4.attn.W_O" } ], "l:162": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 1 ], "matrix": "L0.mlp.W_out" } ], "l:163": [ { "axis0": [ 0, 1 ], "axis1": [ 65, 1 ], "matrix": "L0.mlp.W_out" } ], "l:165": [ { "axis0": [ 0, 1 ], "axis1": [ 1079, 1 ], "matrix": "L3.mlp.W_out" } ], "l:166": [ { "axis0": [ 0, 1 ], "axis1": [ 924, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1796, 1 ], "axis1": [ 924, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1796, 1 ], "axis1": [ 925, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 924, 1 ], "axis1": [ 1264, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 925, 1 ], "axis1": [ 1264, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1264, 1 ], "matrix": "L4.mlp.W_out" } ], "l:167": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 118, 1 ], "axis1": [ 0, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 480, 3 ], "axis1": [ 0, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 64, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 1691, 3 ], "matrix": "L9.attn.W_O" } ], "l:169": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 477, 3 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1690, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 477, 3 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1690, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1, 3 ], "axis1": [ 480, 3 ], "matrix": "L8.mlp.W_out" } ], "l:17": [ { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 987, 1 ], "axis1": [ 448, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 836, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 836, 1 ], "matrix": "L3.mlp.W_out" } ], "l:171": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1690, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 43, 1 ], "axis1": [ 3, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 48, 1 ], "axis1": [ 3, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 986, 1 ], "axis1": [ 3, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 3, 2 ], "axis1": [ 1690, 1 ], "matrix": "L4.mlp.W_out" } ], "l:173": [ { "axis0": [ 0, 1 ], "axis1": [ 37, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 37, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 986, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1662, 1 ], "axis1": [ 37, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 37, 2 ], "axis1": [ 986, 1 ], "matrix": "L2.mlp.W_out" } ], "l:174": [ { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 35, 1 ], "axis1": [ 1088, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 37, 2 ], "axis1": [ 1088, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1088, 64 ], "axis1": [ 477, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 477, 3 ], "matrix": "L8.mlp.W_out" } ], "l:175": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 37, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 34, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 2045, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1, 12 ], "axis1": [ 35, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 1, 12 ], "axis1": [ 37, 2 ], "matrix": "L7.mlp.W_out" } ], "l:177": [ { "axis0": [ 0, 1 ], "axis1": [ 1027, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1183, 1 ], "axis1": [ 1027, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1183, 1 ], "axis1": [ 1028, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1027, 1 ], "axis1": [ 2045, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1028, 1 ], "axis1": [ 2045, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1027, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1028, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 2045, 1 ], "matrix": "L6.mlp.W_out" } ], "l:178": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 2045, 1 ], "axis1": [ 64, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 34, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 34, 1 ], "matrix": "L7.mlp.W_out" } ], "l:18": [ { "axis0": [ 0, 1 ], "axis1": [ 39, 45 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 39, 45 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 987, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1635, 1 ], "axis1": [ 39, 45 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 39, 45 ], "axis1": [ 987, 1 ], "matrix": "L2.mlp.W_out" } ], "l:180": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 4 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 77, 4 ], "axis1": [ 5, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1796, 1 ], "axis1": [ 5, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 77, 4 ], "axis1": [ 5, 4 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1796, 1 ], "axis1": [ 5, 4 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 5, 4 ], "axis1": [ 64, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 5, 4 ], "axis1": [ 1691, 3 ], "matrix": "L4.mlp.W_out" } ], "l:182": [ { "axis0": [ 0, 1 ], "axis1": [ 77, 4 ], "matrix": "L0.mlp.W_out" } ], "l:183": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 66 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 66 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 1, 66 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 979, 3 ], "axis1": [ 1, 66 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 983, 5 ], "axis1": [ 1, 66 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 993, 25 ], "axis1": [ 1, 66 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1690, 1 ], "axis1": [ 1, 66 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1, 33 ], "axis1": [ 1, 66 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 979, 3 ], "axis1": [ 1, 66 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 983, 5 ], "axis1": [ 1, 66 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 993, 25 ], "axis1": [ 1, 66 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1690, 1 ], "axis1": [ 1, 66 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1, 66 ], "axis1": [ 296, 33 ], "matrix": "L9.mlp.W_out" } ], "l:185": [ { "axis0": [ 0, 1 ], "axis1": [ 979, 3 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 983, 5 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 993, 25 ], "matrix": "L8.mlp.W_out" } ], "l:186": [ { "axis0": [ 0, 1 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 50, 2 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 53, 5 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 72, 12 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 143, 9 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 158, 3 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 229, 1 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 332, 1 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 50, 2 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 53, 5 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 72, 12 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 143, 9 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 158, 3 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 229, 1 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 332, 1 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 95, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 95, 33 ], "axis1": [ 35, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 95, 33 ], "axis1": [ 228, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 95, 33 ], "axis1": [ 327, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 95, 33 ], "axis1": [ 869, 8 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 95, 33 ], "axis1": [ 878, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 95, 33 ], "axis1": [ 883, 5 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 95, 33 ], "axis1": [ 965, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 95, 33 ], "axis1": [ 972, 10 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 95, 33 ], "axis1": [ 984, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 95, 33 ], "axis1": [ 1006, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 95, 33 ], "axis1": [ 1009, 1 ], "matrix": "L12.mlp.W_out" } ], "l:188": [ { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 675, 3 ], "axis1": [ 320, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 50, 2 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 53, 5 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 72, 12 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 143, 9 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 158, 3 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 229, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 332, 1 ], "matrix": "L12.attn.W_O" } ], "l:189": [ { "axis0": [ 0, 1 ], "axis1": [ 901, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 901, 300 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 675, 3 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 34, 11 ], "axis1": [ 901, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 46, 13 ], "axis1": [ 901, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 61, 3 ], "axis1": [ 901, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 65, 11 ], "axis1": [ 901, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 172, 2 ], "axis1": [ 901, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 196, 1 ], "axis1": [ 901, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 199, 1 ], "axis1": [ 901, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 223, 12 ], "axis1": [ 901, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 254, 13 ], "axis1": [ 901, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 901, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1150, 1 ], "axis1": [ 901, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 901, 300 ], "axis1": [ 675, 3 ], "matrix": "L11.mlp.W_out" } ], "l:19": [ { "axis0": [ 83, 1 ], "axis1": [ 128, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 832, 1 ], "axis1": [ 128, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1832, 1 ], "axis1": [ 128, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 1635, 1 ], "matrix": "L2.attn.W_O" } ], "l:191": [ { "axis0": [ 0, 1 ], "axis1": [ 37, 6 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 37, 3 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 1, 33 ], "axis1": [ 40, 3 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 37, 3 ], "axis1": [ 55, 3 ], "diagonal": true, "matrix": "L0.mlp.W_out" }, { "axis0": [ 40, 3 ], "axis1": [ 55, 3 ], "diagonal": true, "matrix": "L0.mlp.W_out" } ], "l:192": [ { "axis0": [ 119, 1 ], "axis1": [ 128, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 1018, 1 ], "axis1": [ 128, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 41, 1 ], "axis1": [ 128, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 1019, 1 ], "axis1": [ 128, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 34, 2 ], "axis1": [ 128, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 37, 4 ], "axis1": [ 128, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 42, 2 ], "axis1": [ 128, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 50, 1 ], "axis1": [ 128, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 68, 1 ], "axis1": [ 128, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 70, 1 ], "axis1": [ 128, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 81, 1 ], "axis1": [ 128, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 84, 1 ], "axis1": [ 128, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 92, 20 ], "axis1": [ 128, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 35, 6 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 42, 3 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 46, 9 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 58, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 61, 3 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 65, 11 ], "matrix": "L9.attn.W_O" } ], "l:194": [ { "axis0": [ 0, 1 ], "axis1": [ 1018, 1 ], "matrix": "L8.mlp.W_out" } ], "l:195": [ { "axis0": [ 0, 1 ], "axis1": [ 119, 1 ], "matrix": "L0.mlp.W_out" } ], "l:197": [ { "axis0": [ 0, 1 ], "axis1": [ 1019, 1 ], "matrix": "L8.mlp.W_out" } ], "l:198": [ { "axis0": [ 68, 1 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 1643, 1 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 1703, 1 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 1706, 1 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 1800, 4 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 1190, 33 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 34, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 37, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 42, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 50, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 68, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 70, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 81, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 84, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 92, 20 ], "matrix": "L8.attn.W_O" } ], "l:199": [ { "axis0": [ 0, 1 ], "axis1": [ 9, 9 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 9 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 68, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1643, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1703, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1706, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1677, 1 ], "axis1": [ 9, 9 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 9, 9 ], "axis1": [ 68, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 9, 9 ], "axis1": [ 1643, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 9, 9 ], "axis1": [ 1703, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 9, 9 ], "axis1": [ 1706, 1 ], "matrix": "L4.mlp.W_out" } ], "l:2": [ { "axis0": [ 828, 2 ], "axis1": [ 1984, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 41, 1 ], "axis1": [ 1984, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 830, 1 ], "axis1": [ 1984, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 41, 1 ], "axis1": [ 1984, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1984, 64 ], "axis1": [ 1790, 1 ], "matrix": "L1.attn.W_O" } ], "l:20": [ { "axis0": [ 0, 1 ], "axis1": [ 83, 1 ], "matrix": "L0.mlp.W_out" } ], "l:200": [ { "axis0": [ 1184, 2 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 41, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 1186, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 34, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 64, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 172, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 223, 12 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 254, 17 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 328, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 34, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 41, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 172, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 196, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 199, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 223, 12 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 254, 13 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 331, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 1150, 1 ], "matrix": "L11.attn.W_O" } ], "l:202": [ { "axis0": [ 0, 1 ], "axis1": [ 1184, 1 ], "matrix": "L7.mlp.W_out" } ], "l:203": [ { "axis0": [ 0, 1 ], "axis1": [ 1185, 1 ], "matrix": "L7.mlp.W_out" } ], "l:205": [ { "axis0": [ 0, 1 ], "axis1": [ 1186, 1 ], "matrix": "L7.mlp.W_out" } ], "l:206": [ { "axis0": [ 335, 4 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 64, 1 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 1691, 3 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 296, 33 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 34, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 64, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 172, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 223, 12 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 254, 17 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 328, 1 ], "matrix": "L10.attn.W_O" } ], "l:207": [ { "axis0": [ 0, 1 ], "axis1": [ 80, 9 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 80, 9 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 335, 4 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 34, 1 ], "axis1": [ 80, 9 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 80, 9 ], "axis1": [ 335, 4 ], "matrix": "L9.mlp.W_out" } ], "l:208": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 33 ], "matrix": "L10.mlp.W_up" }, { "axis0": [ 1244, 33 ], "axis1": [ 3, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 3, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 1244, 33 ], "axis1": [ 3, 33 ], "matrix": "L10.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 3, 33 ], "matrix": "L10.mlp.W_up" }, { "axis0": [ 3, 33 ], "axis1": [ 444, 33 ], "matrix": "L10.mlp.W_out" } ], "l:21": [ { "axis0": [ 70, 2 ], "axis1": [ 64, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 44, 1 ], "axis1": [ 64, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 72, 1 ], "axis1": [ 64, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 44, 1 ], "axis1": [ 64, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 832, 1 ], "matrix": "L1.attn.W_O" } ], "l:210": [ { "axis0": [ 0, 1 ], "axis1": [ 1244, 33 ], "matrix": "L9.mlp.W_out" } ], "l:211": [ { "axis0": [ 0, 1 ], "axis1": [ 333, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 333, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 34, 14 ], "axis1": [ 333, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 49, 18 ], "axis1": [ 333, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 333, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1041, 1 ], "axis1": [ 333, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 34, 14 ], "axis1": [ 333, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 49, 18 ], "axis1": [ 333, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 69, 1 ], "axis1": [ 333, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1041, 1 ], "axis1": [ 333, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 333, 33 ], "axis1": [ 1400, 32 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 333, 33 ], "axis1": [ 1913, 1 ], "matrix": "L15.mlp.W_out" } ], "l:213": [ { "axis0": [ 0, 1 ], "axis1": [ 878, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 878, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1041, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1018, 2 ], "axis1": [ 878, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 878, 2 ], "axis1": [ 1041, 1 ], "matrix": "L4.mlp.W_out" } ], "l:215": [ { "axis0": [ 0, 1 ], "axis1": [ 172, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 172, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 172, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1018, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 172, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 172, 2 ], "axis1": [ 1018, 1 ], "matrix": "L3.mlp.W_out" } ], "l:216": [ { "axis0": [ 0, 1 ], "axis1": [ 174, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 174, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 174, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1019, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 174, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 174, 2 ], "axis1": [ 1019, 1 ], "matrix": "L3.mlp.W_out" } ], "l:217": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 198, 1 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 203, 1 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 983, 1 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 986, 2 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 993, 6 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1000, 6 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1010, 9 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1032, 5 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1569, 12 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1590, 21 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 34, 14 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 49, 18 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 69, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 1037, 1 ], "axis1": [ 64, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1042, 1 ], "axis1": [ 64, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1611, 27 ], "axis1": [ 64, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1639, 6 ], "axis1": [ 64, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 34, 14 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 49, 18 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 69, 1 ], "matrix": "L15.attn.W_O" } ], "l:219": [ { "axis0": [ 0, 1 ], "axis1": [ 392, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 392, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1453, 4 ], "axis1": [ 392, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1458, 29 ], "axis1": [ 392, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 392, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1453, 4 ], "axis1": [ 392, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1458, 29 ], "axis1": [ 392, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 392, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 392, 33 ], "axis1": [ 1569, 12 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 392, 33 ], "axis1": [ 1590, 21 ], "matrix": "L14.mlp.W_out" } ], "l:221": [ { "axis0": [ 0, 1 ], "axis1": [ 866, 10 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 866, 10 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1453, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1458, 29 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 76, 1 ], "axis1": [ 866, 10 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 866, 10 ], "axis1": [ 1453, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 866, 10 ], "axis1": [ 1458, 29 ], "matrix": "L13.mlp.W_out" } ], "l:222": [ { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 59, 1 ], "axis1": [ 768, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1243, 1 ], "axis1": [ 768, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 76, 1 ], "matrix": "L13.attn.W_O" } ], "l:223": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 963, 1 ], "axis1": [ 0, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 1243, 1 ], "matrix": "L12.attn.W_O" } ], "l:224": [ { "axis0": [ 0, 1 ], "axis1": [ 316, 2 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 76, 16 ], "axis1": [ 316, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 109, 6 ], "axis1": [ 316, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 119, 1 ], "axis1": [ 316, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 133, 7 ], "axis1": [ 316, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 143, 3 ], "axis1": [ 316, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 76, 16 ], "axis1": [ 317, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 109, 6 ], "axis1": [ 317, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 119, 1 ], "axis1": [ 317, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 133, 7 ], "axis1": [ 317, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 143, 3 ], "axis1": [ 317, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 316, 1 ], "axis1": [ 1243, 1 ], "diagonal": true, "matrix": "L9.mlp.W_out" }, { "axis0": [ 317, 1 ], "axis1": [ 1243, 1 ], "diagonal": true, "matrix": "L9.mlp.W_out" } ], "l:225": [ { "axis0": [ 1021, 1 ], "axis1": [ 192, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 1181, 1 ], "axis1": [ 192, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 41, 1 ], "axis1": [ 192, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 1182, 1 ], "axis1": [ 192, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 112, 6 ], "axis1": [ 192, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 120, 27 ], "axis1": [ 192, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 76, 16 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 109, 6 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 119, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 133, 7 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 143, 3 ], "matrix": "L9.attn.W_O" } ], "l:227": [ { "axis0": [ 0, 1 ], "axis1": [ 1021, 1 ], "matrix": "L8.mlp.W_out" } ], "l:228": [ { "axis0": [ 0, 1 ], "axis1": [ 1181, 1 ], "matrix": "L7.mlp.W_out" } ], "l:23": [ { "axis0": [ 0, 1 ], "axis1": [ 70, 1 ], "matrix": "L0.mlp.W_out" } ], "l:230": [ { "axis0": [ 0, 1 ], "axis1": [ 1182, 1 ], "matrix": "L7.mlp.W_out" } ], "l:231": [ { "axis0": [ 39, 2 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 42, 2 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 1800, 4 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 1190, 33 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 112, 6 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 120, 27 ], "matrix": "L8.attn.W_O" } ], "l:232": [ { "axis0": [ 0, 1 ], "axis1": [ 13, 13 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 13, 13 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 39, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 42, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 1677, 1 ], "axis1": [ 13, 13 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 13, 13 ], "axis1": [ 39, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 13, 13 ], "axis1": [ 42, 2 ], "matrix": "L7.mlp.W_out" } ], "l:233": [ { "axis0": [ 0, 1 ], "axis1": [ 1875, 2 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 267, 33 ], "axis1": [ 1875, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 267, 33 ], "axis1": [ 1876, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1875, 1 ], "axis1": [ 963, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1876, 1 ], "axis1": [ 963, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" } ], "l:234": [ { "axis0": [ 1635, 1 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 1724, 1 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 41, 1 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 1725, 1 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 271, 33 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 267, 33 ], "matrix": "L11.attn.W_O" } ], "l:236": [ { "axis0": [ 0, 1 ], "axis1": [ 1635, 1 ], "matrix": "L7.mlp.W_out" } ], "l:237": [ { "axis0": [ 0, 1 ], "axis1": [ 1724, 1 ], "matrix": "L7.mlp.W_out" } ], "l:239": [ { "axis0": [ 0, 1 ], "axis1": [ 1725, 1 ], "matrix": "L7.mlp.W_out" } ], "l:24": [ { "axis0": [ 0, 1 ], "axis1": [ 71, 1 ], "matrix": "L0.mlp.W_out" } ], "l:240": [ { "axis0": [ 329, 1 ], "axis1": [ 64, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 332, 3 ], "axis1": [ 64, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 64, 1 ], "axis1": [ 64, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 1691, 3 ], "axis1": [ 64, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 296, 33 ], "axis1": [ 64, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 271, 33 ], "matrix": "L10.attn.W_O" } ], "l:241": [ { "axis0": [ 0, 1 ], "axis1": [ 67, 13 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 67, 13 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 329, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 332, 3 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 34, 1 ], "axis1": [ 67, 13 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 67, 13 ], "axis1": [ 329, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 67, 13 ], "axis1": [ 332, 3 ], "matrix": "L9.mlp.W_out" } ], "l:242": [ { "axis0": [ 0, 1 ], "axis1": [ 45, 2 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 45, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 1, 33 ], "axis1": [ 46, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 45, 1 ], "axis1": [ 59, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" }, { "axis0": [ 46, 1 ], "axis1": [ 59, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" } ], "l:243": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 34, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 36, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 39, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 43, 2 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 161, 4 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 168, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 171, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 177, 8 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 195, 3 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 199, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 222, 6 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 231, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 255, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 34, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 36, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 39, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 41, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 43, 2 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 49, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 161, 4 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 168, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 171, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 177, 8 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 195, 3 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 199, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 222, 6 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 231, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 255, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 128, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 128, 33 ], "axis1": [ 198, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 128, 33 ], "axis1": [ 203, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 128, 33 ], "axis1": [ 983, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 128, 33 ], "axis1": [ 986, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 128, 33 ], "axis1": [ 993, 6 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 128, 33 ], "axis1": [ 1000, 6 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 128, 33 ], "axis1": [ 1010, 9 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 128, 33 ], "axis1": [ 1032, 6 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 128, 33 ], "axis1": [ 1042, 1 ], "matrix": "L12.mlp.W_out" } ], "l:245": [ { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 678, 3 ], "axis1": [ 384, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 34, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 36, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 39, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 41, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 43, 2 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 49, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 161, 4 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 168, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 171, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 177, 8 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 195, 3 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 199, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 222, 6 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 231, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 255, 1 ], "matrix": "L12.attn.W_O" } ], "l:246": [ { "axis0": [ 0, 1 ], "axis1": [ 1201, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1201, 300 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 678, 3 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 55, 3 ], "axis1": [ 1201, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 76, 16 ], "axis1": [ 1201, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 109, 6 ], "axis1": [ 1201, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 119, 1 ], "axis1": [ 1201, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 133, 7 ], "axis1": [ 1201, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 143, 3 ], "axis1": [ 1201, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 267, 33 ], "axis1": [ 1201, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1201, 300 ], "axis1": [ 678, 3 ], "matrix": "L11.mlp.W_out" } ], "l:248": [ { "axis0": [ 0, 1 ], "axis1": [ 425, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 425, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1419, 13 ], "axis1": [ 425, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1433, 20 ], "axis1": [ 425, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 425, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1419, 13 ], "axis1": [ 425, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1433, 20 ], "axis1": [ 425, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 425, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 425, 33 ], "axis1": [ 1611, 27 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 425, 33 ], "axis1": [ 1639, 6 ], "matrix": "L14.mlp.W_out" } ], "l:250": [ { "axis0": [ 0, 1 ], "axis1": [ 852, 14 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 852, 14 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1419, 13 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1433, 20 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 78, 1 ], "axis1": [ 852, 14 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 852, 14 ], "axis1": [ 1419, 13 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 852, 14 ], "axis1": [ 1433, 20 ], "matrix": "L13.mlp.W_out" } ], "l:251": [ { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 60, 1 ], "axis1": [ 896, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1084, 1 ], "axis1": [ 896, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 78, 1 ], "matrix": "L13.attn.W_O" } ], "l:252": [ { "axis0": [ 0, 1 ], "axis1": [ 194, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 131, 2 ], "axis1": [ 194, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 671, 4 ], "axis1": [ 194, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 131, 2 ], "axis1": [ 195, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 671, 4 ], "axis1": [ 195, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 194, 1 ], "axis1": [ 1084, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 195, 1 ], "axis1": [ 1084, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" } ], "l:254": [ { "axis0": [ 0, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 64, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 76, 16 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 109, 6 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 119, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 133, 7 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 143, 3 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 300, 21 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 322, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 332, 3 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 336, 3 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 501, 100 ], "axis1": [ 131, 2 ], "matrix": "L11.mlp.W_out" } ], "l:256": [ { "axis0": [ 0, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 146, 6 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 158, 7 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 168, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 171, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 174, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 177, 8 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 186, 6 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 195, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 267, 33 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 701, 100 ], "axis1": [ 671, 2 ], "matrix": "L11.mlp.W_out" } ], "l:258": [ { "axis0": [ 0, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 64, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 146, 6 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 158, 7 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 168, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 171, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 174, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 177, 8 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 186, 6 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 195, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 300, 21 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 322, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 332, 3 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 336, 3 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 801, 100 ], "axis1": [ 673, 2 ], "matrix": "L11.mlp.W_out" } ], "l:26": [ { "axis0": [ 0, 1 ], "axis1": [ 16, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 16, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 44, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 33 ], "axis1": [ 16, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 16, 1 ], "axis1": [ 44, 1 ], "matrix": "L0.mlp.W_out" } ], "l:260": [ { "axis0": [ 0, 1 ], "axis1": [ 47, 2 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 47, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 1, 33 ], "axis1": [ 48, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 47, 1 ], "axis1": [ 60, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" }, { "axis0": [ 48, 1 ], "axis1": [ 60, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" } ], "l:261": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1399, 1 ], "axis1": [ 1, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1574, 7 ], "axis1": [ 1, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1587, 26 ], "axis1": [ 1, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1399, 1 ], "axis1": [ 1, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1574, 7 ], "axis1": [ 1, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1587, 26 ], "axis1": [ 1, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 272, 33 ], "matrix": "L16.mlp.W_out" } ], "l:263": [ { "axis0": [ 0, 1 ], "axis1": [ 331, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 331, 2 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1399, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 1020, 1 ], "axis1": [ 331, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1831, 1 ], "axis1": [ 331, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 331, 2 ], "axis1": [ 1399, 1 ], "matrix": "L15.mlp.W_out" } ], "l:265": [ { "axis0": [ 0, 1 ], "axis1": [ 176, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 176, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 176, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1020, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 176, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 176, 2 ], "axis1": [ 1020, 1 ], "matrix": "L3.mlp.W_out" } ], "l:266": [ { "axis0": [ 0, 1 ], "axis1": [ 178, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 178, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 178, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1831, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 178, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 178, 2 ], "axis1": [ 1831, 1 ], "matrix": "L3.mlp.W_out" } ], "l:267": [ { "axis0": [ 0, 1 ], "axis1": [ 432, 66 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1026, 1 ], "axis1": [ 432, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1038, 3 ], "axis1": [ 432, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1043, 3 ], "axis1": [ 432, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1047, 2 ], "axis1": [ 432, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1052, 8 ], "axis1": [ 432, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1068, 13 ], "axis1": [ 432, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1082, 2 ], "axis1": [ 432, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1087, 1 ], "axis1": [ 432, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1645, 33 ], "axis1": [ 432, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1680, 1 ], "axis1": [ 432, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1683, 21 ], "axis1": [ 432, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1705, 11 ], "axis1": [ 432, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1026, 1 ], "axis1": [ 465, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1038, 3 ], "axis1": [ 465, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1043, 3 ], "axis1": [ 465, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1047, 2 ], "axis1": [ 465, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1052, 8 ], "axis1": [ 465, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1068, 13 ], "axis1": [ 465, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1082, 2 ], "axis1": [ 465, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1087, 1 ], "axis1": [ 465, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1645, 33 ], "axis1": [ 465, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1680, 1 ], "axis1": [ 465, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1683, 21 ], "axis1": [ 465, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1705, 11 ], "axis1": [ 465, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 432, 7 ], "axis1": [ 1574, 7 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 439, 26 ], "axis1": [ 1587, 26 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 465, 7 ], "axis1": [ 1574, 7 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 472, 26 ], "axis1": [ 1587, 26 ], "diagonal": true, "matrix": "L15.mlp.W_out" } ], "l:269": [ { "axis0": [ 0, 1 ], "axis1": [ 458, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 458, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1386, 33 ], "axis1": [ 458, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 458, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1386, 33 ], "axis1": [ 458, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 458, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 458, 33 ], "axis1": [ 1645, 33 ], "matrix": "L14.mlp.W_out" } ], "l:27": [ { "axis0": [ 0, 1 ], "axis1": [ 72, 1 ], "matrix": "L0.mlp.W_out" } ], "l:271": [ { "axis0": [ 0, 1 ], "axis1": [ 842, 10 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 842, 10 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1386, 33 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 77, 1 ], "axis1": [ 842, 10 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 842, 10 ], "axis1": [ 1386, 33 ], "matrix": "L13.mlp.W_out" } ], "l:272": [ { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 59, 1 ], "axis1": [ 832, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 212, 1 ], "axis1": [ 832, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 77, 1 ], "matrix": "L13.attn.W_O" } ], "l:273": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 962, 1 ], "axis1": [ 64, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 212, 1 ], "matrix": "L12.attn.W_O" } ], "l:274": [ { "axis0": [ 0, 1 ], "axis1": [ 314, 2 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 35, 6 ], "axis1": [ 314, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 42, 3 ], "axis1": [ 314, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 46, 9 ], "axis1": [ 314, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 314, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 61, 3 ], "axis1": [ 314, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 65, 11 ], "axis1": [ 314, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 35, 6 ], "axis1": [ 315, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 42, 3 ], "axis1": [ 315, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 46, 9 ], "axis1": [ 315, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 315, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 61, 3 ], "axis1": [ 315, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 65, 11 ], "axis1": [ 315, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 314, 1 ], "axis1": [ 212, 1 ], "diagonal": true, "matrix": "L9.mlp.W_out" }, { "axis0": [ 315, 1 ], "axis1": [ 212, 1 ], "diagonal": true, "matrix": "L9.mlp.W_out" } ], "l:275": [ { "axis0": [ 0, 1 ], "axis1": [ 1873, 2 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 34, 1 ], "axis1": [ 1873, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 1873, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 172, 2 ], "axis1": [ 1873, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 196, 1 ], "axis1": [ 1873, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 199, 1 ], "axis1": [ 1873, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 223, 12 ], "axis1": [ 1873, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 254, 13 ], "axis1": [ 1873, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 1873, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1150, 1 ], "axis1": [ 1873, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 34, 1 ], "axis1": [ 1874, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 1874, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 172, 2 ], "axis1": [ 1874, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 196, 1 ], "axis1": [ 1874, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 199, 1 ], "axis1": [ 1874, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 223, 12 ], "axis1": [ 1874, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 254, 13 ], "axis1": [ 1874, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 1874, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1150, 1 ], "axis1": [ 1874, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1873, 1 ], "axis1": [ 962, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1874, 1 ], "axis1": [ 962, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" } ], "l:276": [ { "axis0": [ 0, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 67, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 230, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 254, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 256, 28 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 1150, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 173, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 230, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 254, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 256, 28 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 1150, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 161, 33 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 161, 33 ], "axis1": [ 1026, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 161, 33 ], "axis1": [ 1038, 3 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 161, 33 ], "axis1": [ 1043, 3 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 161, 33 ], "axis1": [ 1047, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 161, 33 ], "axis1": [ 1052, 8 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 161, 33 ], "axis1": [ 1068, 13 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 161, 33 ], "axis1": [ 1082, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 161, 33 ], "axis1": [ 1087, 1 ], "matrix": "L12.mlp.W_out" } ], "l:278": [ { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 681, 3 ], "axis1": [ 448, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 67, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 173, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 230, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 254, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 256, 28 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1150, 1 ], "matrix": "L12.attn.W_O" } ], "l:279": [ { "axis0": [ 0, 1 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 681, 3 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 55, 3 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 64, 1 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 146, 6 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 158, 7 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 168, 1 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 171, 1 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 174, 1 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 177, 8 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 186, 6 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 195, 1 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 300, 21 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 322, 1 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 332, 3 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 336, 3 ], "axis1": [ 1501, 300 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1501, 300 ], "axis1": [ 681, 3 ], "matrix": "L11.mlp.W_out" } ], "l:28": [ { "axis0": [ 0, 1 ], "axis1": [ 70, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 44, 1 ], "axis1": [ 70, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 44, 1 ], "axis1": [ 71, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 70, 1 ], "axis1": [ 1832, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 71, 1 ], "axis1": [ 1832, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1832, 1 ], "matrix": "L1.mlp.W_out" } ], "l:281": [ { "axis0": [ 0, 1 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1290, 24 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1332, 1 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1357, 1 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1369, 1 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1380, 6 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1290, 24 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1332, 1 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1357, 1 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1369, 1 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1380, 6 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 491, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 491, 33 ], "axis1": [ 1680, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 491, 33 ], "axis1": [ 1683, 21 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 491, 33 ], "axis1": [ 1705, 11 ], "matrix": "L14.mlp.W_out" } ], "l:283": [ { "axis0": [ 0, 1 ], "axis1": [ 820, 22 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 820, 22 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1290, 24 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1332, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1357, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1369, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1380, 6 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 79, 1 ], "axis1": [ 820, 22 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 820, 22 ], "axis1": [ 1290, 24 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 820, 22 ], "axis1": [ 1332, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 820, 22 ], "axis1": [ 1357, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 820, 22 ], "axis1": [ 1369, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 820, 22 ], "axis1": [ 1380, 6 ], "matrix": "L13.mlp.W_out" } ], "l:284": [ { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 60, 1 ], "axis1": [ 960, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1085, 1 ], "axis1": [ 960, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 960, 64 ], "axis1": [ 79, 1 ], "matrix": "L13.attn.W_O" } ], "l:285": [ { "axis0": [ 0, 1 ], "axis1": [ 196, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 130, 3 ], "axis1": [ 196, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 200, 1 ], "axis1": [ 196, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 344, 1 ], "axis1": [ 196, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 669, 4 ], "axis1": [ 196, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 1025, 1 ], "axis1": [ 196, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 130, 3 ], "axis1": [ 197, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 200, 1 ], "axis1": [ 197, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 344, 1 ], "axis1": [ 197, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 669, 4 ], "axis1": [ 197, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 1025, 1 ], "axis1": [ 197, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 196, 1 ], "axis1": [ 1085, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 197, 1 ], "axis1": [ 1085, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" } ], "l:287": [ { "axis0": [ 0, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 35, 6 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 42, 3 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 46, 9 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 61, 15 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 300, 21 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 322, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 332, 3 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 336, 3 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 201, 100 ], "axis1": [ 344, 1 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 201, 100 ], "axis1": [ 1025, 1 ], "matrix": "L11.mlp.W_out" } ], "l:289": [ { "axis0": [ 0, 1 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 76, 16 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 109, 6 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 119, 1 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 133, 7 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 143, 3 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 267, 33 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 401, 100 ], "axis1": [ 130, 1 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 401, 100 ], "axis1": [ 200, 1 ], "matrix": "L11.mlp.W_out" } ], "l:29": [ { "axis0": [ 0, 1 ], "axis1": [ 166, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 166, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 166, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1005, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 166, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 166, 2 ], "axis1": [ 1005, 1 ], "matrix": "L3.mlp.W_out" } ], "l:291": [ { "axis0": [ 0, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 34, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 146, 6 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 158, 7 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 168, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 171, 4 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 177, 8 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 186, 6 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 195, 3 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 199, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 222, 13 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 254, 13 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1150, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 601, 100 ], "axis1": [ 669, 2 ], "matrix": "L11.mlp.W_out" } ], "l:293": [ { "axis0": [ 0, 1 ], "axis1": [ 34, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 34, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 34, 33 ], "axis1": [ 34, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1046, 1 ], "axis1": [ 34, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 34, 33 ], "axis1": [ 34, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1046, 1 ], "axis1": [ 34, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 34, 33 ], "axis1": [ 305, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 34, 33 ], "axis1": [ 308, 4 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 34, 33 ], "axis1": [ 313, 4 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 34, 33 ], "axis1": [ 318, 23 ], "matrix": "L16.mlp.W_out" } ], "l:295": [ { "axis0": [ 0, 1 ], "axis1": [ 880, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 880, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1046, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1022, 1 ], "axis1": [ 880, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1026, 1 ], "axis1": [ 880, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 880, 2 ], "axis1": [ 1046, 1 ], "matrix": "L4.mlp.W_out" } ], "l:297": [ { "axis0": [ 0, 1 ], "axis1": [ 180, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 180, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 180, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1022, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 180, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 180, 2 ], "axis1": [ 1022, 1 ], "matrix": "L3.mlp.W_out" } ], "l:298": [ { "axis0": [ 0, 1 ], "axis1": [ 182, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 182, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 182, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1026, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 182, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 182, 2 ], "axis1": [ 1026, 1 ], "matrix": "L3.mlp.W_out" } ], "l:299": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 92, 2 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 146, 6 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 158, 7 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 168, 1 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 171, 4 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 177, 9 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 192, 1 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 195, 3 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 343, 1 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 347, 6 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 419, 20 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 440, 4 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 34, 33 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 48, 1 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 198, 4 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 203, 1 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 222, 6 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 229, 3 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 237, 4 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 244, 3 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 254, 13 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 34, 33 ], "matrix": "L16.attn.W_O" } ], "l:30": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 75, 1 ], "axis1": [ 128, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1215, 49 ], "axis1": [ 128, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1265, 14 ], "axis1": [ 128, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 37, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 42, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 44, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 48, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 51, 4 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 58, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 61, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 67, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 72, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 75, 6 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 137, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 168, 12 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 1279, 35 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 37, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 42, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 44, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 48, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 51, 4 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 58, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 61, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 67, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 72, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 75, 6 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 137, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 168, 12 ], "matrix": "L5.attn.W_O" } ], "l:301": [ { "axis0": [ 0, 1 ], "axis1": [ 110, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 110, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1102, 9 ], "axis1": [ 110, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1112, 1 ], "axis1": [ 110, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1187, 3 ], "axis1": [ 110, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1223, 20 ], "axis1": [ 110, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 110, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1102, 9 ], "axis1": [ 110, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1112, 1 ], "axis1": [ 110, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1187, 3 ], "axis1": [ 110, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1223, 20 ], "axis1": [ 110, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 110, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 110, 33 ], "axis1": [ 92, 2 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 110, 33 ], "axis1": [ 343, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 110, 33 ], "axis1": [ 347, 6 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 110, 33 ], "axis1": [ 419, 20 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 110, 33 ], "axis1": [ 440, 4 ], "matrix": "L9.mlp.W_out" } ], "l:303": [ { "axis0": [ 0, 1 ], "axis1": [ 1102, 9 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1112, 1 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1187, 3 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1223, 20 ], "matrix": "L8.mlp.W_out" } ], "l:304": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1501, 19 ], "axis1": [ 1, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1554, 4 ], "axis1": [ 1, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1559, 10 ], "axis1": [ 1, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 1, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1501, 19 ], "axis1": [ 1, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1554, 4 ], "axis1": [ 1, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1559, 10 ], "axis1": [ 1, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 1, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 146, 6 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 1, 33 ], "axis1": [ 158, 7 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 1, 33 ], "axis1": [ 168, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 1, 33 ], "axis1": [ 171, 4 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 1, 33 ], "axis1": [ 177, 9 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 1, 33 ], "axis1": [ 192, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 1, 33 ], "axis1": [ 195, 5 ], "matrix": "L15.mlp.W_out" } ], "l:306": [ { "axis0": [ 0, 1 ], "axis1": [ 388, 4 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 388, 4 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1501, 19 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1554, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1559, 10 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 295, 1 ], "axis1": [ 388, 4 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 388, 4 ], "axis1": [ 1501, 19 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 388, 4 ], "axis1": [ 1554, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 388, 4 ], "axis1": [ 1559, 10 ], "matrix": "L14.mlp.W_out" } ], "l:307": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 75, 1 ], "axis1": [ 0, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 295, 1 ], "matrix": "L14.attn.W_O" } ], "l:308": [ { "axis0": [ 0, 1 ], "axis1": [ 21, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 21, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 45, 1 ], "axis1": [ 21, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 245, 1 ], "axis1": [ 21, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 961, 1 ], "axis1": [ 21, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 45, 1 ], "axis1": [ 21, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 245, 1 ], "axis1": [ 21, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 961, 1 ], "axis1": [ 21, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 21, 2 ], "axis1": [ 295, 1 ], "matrix": "L12.mlp.W_out" } ], "l:310": [ { "axis0": [ 0, 1 ], "axis1": [ 17, 2 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 17, 2 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 45, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 34, 1 ], "axis1": [ 17, 2 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 17, 2 ], "axis1": [ 45, 1 ], "matrix": "L0.mlp.W_out" } ], "l:311": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L0.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L0.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 0, 64 ], "matrix": "L0.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 34, 1 ], "matrix": "L0.attn.W_O" } ], "l:312": [ { "axis0": [ 0, 1 ], "axis1": [ 312, 2 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 146, 6 ], "axis1": [ 312, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 158, 7 ], "axis1": [ 312, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 168, 1 ], "axis1": [ 312, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 171, 1 ], "axis1": [ 312, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 174, 1 ], "axis1": [ 312, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 177, 8 ], "axis1": [ 312, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 186, 6 ], "axis1": [ 312, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 195, 1 ], "axis1": [ 312, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 312, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 312, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 146, 6 ], "axis1": [ 313, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 158, 7 ], "axis1": [ 313, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 168, 1 ], "axis1": [ 313, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 171, 1 ], "axis1": [ 313, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 174, 1 ], "axis1": [ 313, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 177, 8 ], "axis1": [ 313, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 186, 6 ], "axis1": [ 313, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 195, 1 ], "axis1": [ 313, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 313, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 313, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 312, 1 ], "axis1": [ 245, 1 ], "diagonal": true, "matrix": "L9.mlp.W_out" }, { "axis0": [ 313, 1 ], "axis1": [ 245, 1 ], "diagonal": true, "matrix": "L9.mlp.W_out" } ], "l:313": [ { "axis0": [ 0, 1 ], "axis1": [ 1871, 2 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 64, 1 ], "axis1": [ 1871, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 300, 21 ], "axis1": [ 1871, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 322, 1 ], "axis1": [ 1871, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 1871, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 1871, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 332, 3 ], "axis1": [ 1871, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 336, 3 ], "axis1": [ 1871, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 64, 1 ], "axis1": [ 1872, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 300, 21 ], "axis1": [ 1872, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 322, 1 ], "axis1": [ 1872, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 1872, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 1872, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 332, 3 ], "axis1": [ 1872, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 336, 3 ], "axis1": [ 1872, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1871, 1 ], "axis1": [ 961, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1872, 1 ], "axis1": [ 961, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" } ], "l:314": [ { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 289, 1 ], "axis1": [ 704, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 75, 1 ], "matrix": "L13.attn.W_O" } ], "l:315": [ { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 45, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 245, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 961, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 45, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 245, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 961, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 9, 2 ], "axis1": [ 289, 1 ], "matrix": "L12.mlp.W_out" } ], "l:317": [ { "axis0": [ 0, 1 ], "axis1": [ 34, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 34, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1257, 33 ], "axis1": [ 34, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 34, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1257, 33 ], "axis1": [ 34, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 34, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 34, 33 ], "axis1": [ 48, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 34, 33 ], "axis1": [ 200, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 34, 33 ], "axis1": [ 203, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 34, 33 ], "axis1": [ 222, 6 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 34, 33 ], "axis1": [ 229, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 34, 33 ], "axis1": [ 237, 4 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 34, 33 ], "axis1": [ 244, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 34, 33 ], "axis1": [ 254, 13 ], "matrix": "L15.mlp.W_out" } ], "l:319": [ { "axis0": [ 0, 1 ], "axis1": [ 798, 22 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 798, 22 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1257, 33 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 80, 1 ], "axis1": [ 798, 22 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 798, 22 ], "axis1": [ 1257, 33 ], "matrix": "L13.mlp.W_out" } ], "l:32": [ { "axis0": [ 0, 1 ], "axis1": [ 214, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 214, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 44, 1 ], "axis1": [ 214, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1085, 26 ], "axis1": [ 214, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1112, 6 ], "axis1": [ 214, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 214, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 44, 1 ], "axis1": [ 214, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1085, 26 ], "axis1": [ 214, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1112, 6 ], "axis1": [ 214, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 214, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 214, 33 ], "axis1": [ 1215, 33 ], "matrix": "L4.mlp.W_out" } ], "l:320": [ { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 60, 1 ], "axis1": [ 1024, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1086, 1 ], "axis1": [ 1024, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1024, 64 ], "axis1": [ 80, 1 ], "matrix": "L13.attn.W_O" } ], "l:321": [ { "axis0": [ 0, 1 ], "axis1": [ 198, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 130, 1 ], "axis1": [ 198, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 185, 1 ], "axis1": [ 198, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 200, 1 ], "axis1": [ 198, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 341, 2 ], "axis1": [ 198, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 344, 2 ], "axis1": [ 198, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 669, 2 ], "axis1": [ 198, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 1025, 1 ], "axis1": [ 198, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 130, 1 ], "axis1": [ 199, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 185, 1 ], "axis1": [ 199, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 200, 1 ], "axis1": [ 199, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 341, 2 ], "axis1": [ 199, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 344, 2 ], "axis1": [ 199, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 669, 2 ], "axis1": [ 199, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 1025, 1 ], "axis1": [ 199, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 198, 1 ], "axis1": [ 1086, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 199, 1 ], "axis1": [ 1086, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" } ], "l:323": [ { "axis0": [ 0, 1 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 35, 6 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 42, 3 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 46, 9 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 61, 3 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 65, 11 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 267, 33 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 101, 100 ], "axis1": [ 341, 2 ], "matrix": "L11.mlp.W_out" } ], "l:325": [ { "axis0": [ 0, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 34, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 76, 16 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 109, 6 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 119, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 133, 7 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 143, 3 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 172, 2 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 196, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 199, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 223, 12 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 254, 13 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1150, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 301, 100 ], "axis1": [ 185, 1 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 301, 100 ], "axis1": [ 345, 1 ], "matrix": "L11.mlp.W_out" } ], "l:327": [ { "axis0": [ 0, 1 ], "axis1": [ 67, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 67, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 67, 33 ], "axis1": [ 67, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1051, 1 ], "axis1": [ 67, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 67, 33 ], "axis1": [ 67, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1051, 1 ], "axis1": [ 67, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 67, 33 ], "axis1": [ 341, 33 ], "matrix": "L16.mlp.W_out" } ], "l:329": [ { "axis0": [ 0, 1 ], "axis1": [ 882, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 882, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1051, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1024, 2 ], "axis1": [ 882, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 882, 2 ], "axis1": [ 1051, 1 ], "matrix": "L4.mlp.W_out" } ], "l:331": [ { "axis0": [ 0, 1 ], "axis1": [ 184, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 184, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 184, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 184, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 184, 2 ], "axis1": [ 1024, 1 ], "matrix": "L3.mlp.W_out" } ], "l:332": [ { "axis0": [ 0, 1 ], "axis1": [ 186, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 186, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 186, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1025, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 186, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 186, 2 ], "axis1": [ 1025, 1 ], "matrix": "L3.mlp.W_out" } ], "l:333": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 267, 27 ], "axis1": [ 128, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 296, 1 ], "axis1": [ 128, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 321, 1 ], "axis1": [ 128, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 323, 1 ], "axis1": [ 128, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 329, 1 ], "axis1": [ 128, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1314, 18 ], "axis1": [ 128, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1333, 14 ], "axis1": [ 128, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1681, 1 ], "axis1": [ 128, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 67, 33 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 332, 1 ], "axis1": [ 192, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1716, 33 ], "axis1": [ 192, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1858, 1 ], "axis1": [ 192, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 67, 33 ], "matrix": "L16.attn.W_O" } ], "l:335": [ { "axis0": [ 0, 1 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 954, 1 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 956, 4 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 961, 4 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 966, 4 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 971, 9 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 981, 1 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 987, 1 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 999, 4 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1818, 4 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1823, 1 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 954, 1 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 956, 4 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 961, 4 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 966, 4 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 971, 9 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 981, 1 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 987, 1 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 999, 4 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1818, 4 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1823, 1 ], "axis1": [ 313, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 313, 33 ], "axis1": [ 1314, 18 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 313, 33 ], "axis1": [ 1333, 14 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 313, 33 ], "axis1": [ 1681, 1 ], "matrix": "L4.mlp.W_out" } ], "l:337": [ { "axis0": [ 0, 1 ], "axis1": [ 138, 26 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 138, 26 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 954, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 956, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 961, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 966, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 971, 9 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 981, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 987, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 999, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1818, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1823, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 1017, 1 ], "axis1": [ 138, 26 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 138, 26 ], "axis1": [ 954, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 138, 26 ], "axis1": [ 956, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 138, 26 ], "axis1": [ 961, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 138, 26 ], "axis1": [ 966, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 138, 26 ], "axis1": [ 971, 9 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 138, 26 ], "axis1": [ 981, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 138, 26 ], "axis1": [ 987, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 138, 26 ], "axis1": [ 999, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 138, 26 ], "axis1": [ 1818, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 138, 26 ], "axis1": [ 1823, 1 ], "matrix": "L3.mlp.W_out" } ], "l:338": [ { "axis0": [ 0, 1 ], "axis1": [ 155, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 72, 1 ], "axis1": [ 155, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 881, 8 ], "axis1": [ 155, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 890, 24 ], "axis1": [ 155, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 72, 1 ], "axis1": [ 156, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 881, 8 ], "axis1": [ 156, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 890, 24 ], "axis1": [ 156, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 155, 1 ], "axis1": [ 1017, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 156, 1 ], "axis1": [ 1017, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" } ], "l:339": [ { "axis0": [ 160, 1 ], "axis1": [ 640, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 160, 1 ], "axis1": [ 640, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 640, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 72, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 881, 8 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 890, 24 ], "matrix": "L2.attn.W_O" } ], "l:34": [ { "axis0": [ 161, 1 ], "axis1": [ 320, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 1082, 1 ], "axis1": [ 320, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 162, 1 ], "axis1": [ 320, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1796, 1 ], "axis1": [ 320, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 38, 1 ], "axis1": [ 320, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 1678, 1 ], "matrix": "L4.attn.W_O" } ], "l:340": [ { "axis0": [ 0, 1 ], "axis1": [ 160, 1 ], "matrix": "L0.mlp.W_out" } ], "l:341": [ { "axis0": [ 0, 1 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 975, 7 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 984, 2 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1006, 4 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1030, 1 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1084, 3 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1088, 1 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1243, 1 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1487, 14 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 975, 7 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 984, 2 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1006, 4 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1030, 1 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1084, 3 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1088, 1 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1243, 1 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1487, 14 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 67, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 67, 33 ], "axis1": [ 267, 27 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 67, 33 ], "axis1": [ 296, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 67, 33 ], "axis1": [ 321, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 67, 33 ], "axis1": [ 323, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 67, 33 ], "axis1": [ 329, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 67, 33 ], "axis1": [ 332, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 67, 33 ], "axis1": [ 1858, 1 ], "matrix": "L15.mlp.W_out" } ], "l:343": [ { "axis0": [ 0, 1 ], "axis1": [ 379, 9 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 379, 9 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 290, 1 ], "axis1": [ 379, 9 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 379, 9 ], "axis1": [ 975, 7 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 379, 9 ], "axis1": [ 984, 2 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 379, 9 ], "axis1": [ 1006, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 379, 9 ], "axis1": [ 1030, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 379, 9 ], "axis1": [ 1084, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 379, 9 ], "axis1": [ 1088, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 379, 9 ], "axis1": [ 1243, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 379, 9 ], "axis1": [ 1487, 14 ], "matrix": "L14.mlp.W_out" } ], "l:344": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 1833, 1 ], "axis1": [ 64, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 290, 1 ], "matrix": "L14.attn.W_O" } ], "l:345": [ { "axis0": [ 0, 1 ], "axis1": [ 979, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 52, 1 ], "axis1": [ 979, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 980, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 979, 1 ], "axis1": [ 290, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 980, 1 ], "axis1": [ 290, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" } ], "l:346": [ { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 246, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 960, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 988, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 246, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 960, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 988, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 11, 2 ], "axis1": [ 290, 1 ], "matrix": "L12.mlp.W_out" } ], "l:348": [ { "axis0": [ 0, 1 ], "axis1": [ 84, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 84, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 988, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1813, 1 ], "axis1": [ 84, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 84, 2 ], "axis1": [ 988, 1 ], "matrix": "L2.mlp.W_out" } ], "l:349": [ { "axis0": [ 0, 1 ], "axis1": [ 32, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 833, 33 ], "axis1": [ 32, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 833, 33 ], "axis1": [ 33, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 32, 1 ], "axis1": [ 1813, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 33, 1 ], "axis1": [ 1813, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:350": [ { "axis0": [ 120, 1 ], "axis1": [ 128, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 120, 1 ], "axis1": [ 128, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 128, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 833, 33 ], "matrix": "L1.attn.W_O" } ], "l:351": [ { "axis0": [ 0, 1 ], "axis1": [ 120, 1 ], "matrix": "L0.mlp.W_out" } ], "l:352": [ { "axis0": [ 0, 1 ], "axis1": [ 310, 2 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 76, 16 ], "axis1": [ 310, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 109, 6 ], "axis1": [ 310, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 119, 1 ], "axis1": [ 310, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 133, 7 ], "axis1": [ 310, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 143, 3 ], "axis1": [ 310, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 76, 16 ], "axis1": [ 311, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 109, 6 ], "axis1": [ 311, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 119, 1 ], "axis1": [ 311, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 133, 7 ], "axis1": [ 311, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 143, 3 ], "axis1": [ 311, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 310, 1 ], "axis1": [ 246, 1 ], "diagonal": true, "matrix": "L9.mlp.W_out" }, { "axis0": [ 311, 1 ], "axis1": [ 246, 1 ], "diagonal": true, "matrix": "L9.mlp.W_out" } ], "l:353": [ { "axis0": [ 0, 1 ], "axis1": [ 1869, 2 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 267, 33 ], "axis1": [ 1869, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 267, 33 ], "axis1": [ 1870, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1869, 1 ], "axis1": [ 960, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1870, 1 ], "axis1": [ 960, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" } ], "l:354": [ { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 174, 1 ], "axis1": [ 448, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 52, 1 ], "matrix": "L13.attn.W_O" } ], "l:355": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 246, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 960, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 988, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 246, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 960, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 988, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 1, 2 ], "axis1": [ 174, 1 ], "matrix": "L12.mlp.W_out" } ], "l:357": [ { "axis0": [ 0, 1 ], "axis1": [ 72, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 58, 1 ], "axis1": [ 72, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 73, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 72, 1 ], "axis1": [ 1833, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 73, 1 ], "axis1": [ 1833, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:358": [ { "axis0": [ 0, 1 ], "axis1": [ 43, 2 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 43, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 1, 33 ], "axis1": [ 44, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 43, 1 ], "axis1": [ 58, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" }, { "axis0": [ 44, 1 ], "axis1": [ 58, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" } ], "l:359": [ { "axis0": [ 0, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 251, 3 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 267, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 271, 2 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 277, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 282, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 284, 2 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 287, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 292, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 307, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 312, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 317, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 321, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 323, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 439, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 514, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1027, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1065, 3 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1092, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1111, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1146, 3 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1151, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1238, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1247, 2 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1253, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 251, 3 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 267, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 271, 2 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 277, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 282, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 284, 2 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 287, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 292, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 307, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 312, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 317, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 321, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 323, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 439, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 514, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1027, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1065, 3 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1092, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1111, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1146, 3 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1151, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1238, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1247, 2 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1253, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 524, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 524, 33 ], "axis1": [ 1716, 33 ], "matrix": "L14.mlp.W_out" } ], "l:36": [ { "axis0": [ 0, 1 ], "axis1": [ 1082, 1 ], "matrix": "L3.mlp.W_out" } ], "l:361": [ { "axis0": [ 0, 1 ], "axis1": [ 784, 14 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 784, 14 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 251, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 267, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 271, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 277, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 282, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 284, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 287, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 292, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 307, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 312, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 317, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 321, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 323, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 439, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 514, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1027, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1065, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1092, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1111, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1146, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1151, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1238, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1247, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1253, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 81, 1 ], "axis1": [ 784, 14 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 784, 14 ], "axis1": [ 251, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 267, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 271, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 277, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 282, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 284, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 287, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 292, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 307, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 312, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 317, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 321, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 323, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 439, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 514, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 1027, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 1065, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 1092, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 1111, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 1146, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 1151, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 1238, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 1247, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 784, 14 ], "axis1": [ 1253, 1 ], "matrix": "L13.mlp.W_out" } ], "l:362": [ { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 60, 1 ], "axis1": [ 1088, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1030, 1 ], "axis1": [ 1088, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1088, 64 ], "axis1": [ 81, 1 ], "matrix": "L13.attn.W_O" } ], "l:363": [ { "axis0": [ 0, 1 ], "axis1": [ 200, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 185, 1 ], "axis1": [ 200, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 339, 4 ], "axis1": [ 200, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 345, 1 ], "axis1": [ 200, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 185, 1 ], "axis1": [ 201, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 339, 4 ], "axis1": [ 201, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 345, 1 ], "axis1": [ 201, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 200, 1 ], "axis1": [ 1030, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 201, 1 ], "axis1": [ 1030, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" } ], "l:365": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 34, 11 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 46, 9 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 61, 3 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 65, 11 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 172, 2 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 196, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 199, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 223, 12 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 254, 13 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1150, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1, 100 ], "axis1": [ 339, 2 ], "matrix": "L11.mlp.W_out" } ], "l:367": [ { "axis0": [ 0, 1 ], "axis1": [ 100, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 100, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 100, 33 ], "axis1": [ 100, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1061, 1 ], "axis1": [ 100, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 100, 33 ], "axis1": [ 100, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1061, 1 ], "axis1": [ 100, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 100, 33 ], "axis1": [ 374, 33 ], "matrix": "L16.mlp.W_out" } ], "l:369": [ { "axis0": [ 0, 1 ], "axis1": [ 884, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 884, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1061, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1027, 1 ], "axis1": [ 884, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1832, 1 ], "axis1": [ 884, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 884, 2 ], "axis1": [ 1061, 1 ], "matrix": "L4.mlp.W_out" } ], "l:37": [ { "axis0": [ 0, 1 ], "axis1": [ 161, 1 ], "matrix": "L0.mlp.W_out" } ], "l:371": [ { "axis0": [ 0, 1 ], "axis1": [ 188, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 188, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 188, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1832, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 188, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 188, 2 ], "axis1": [ 1832, 1 ], "matrix": "L3.mlp.W_out" } ], "l:372": [ { "axis0": [ 0, 1 ], "axis1": [ 190, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 190, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 190, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1027, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 190, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 190, 2 ], "axis1": [ 1027, 1 ], "matrix": "L3.mlp.W_out" } ], "l:373": [ { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 83, 1 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 91, 1 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 94, 2 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 99, 7 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 335, 8 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 344, 2 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 439, 1 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 480, 16 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 514, 1 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 669, 6 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 868, 1 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 881, 2 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 959, 6 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 982, 2 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 986, 8 ], "axis1": [ 256, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 100, 33 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 994, 2 ], "axis1": [ 320, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1749, 33 ], "axis1": [ 320, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 100, 33 ], "matrix": "L16.attn.W_O" } ], "l:375": [ { "axis0": [ 0, 1 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 247, 4 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1060, 1 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1062, 1 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1064, 1 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1226, 12 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1239, 4 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1244, 3 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1249, 4 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1254, 3 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 247, 4 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1060, 1 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1062, 1 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1064, 1 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1226, 12 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1239, 4 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1244, 3 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1249, 4 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1254, 3 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 100, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 100, 33 ], "axis1": [ 83, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 100, 33 ], "axis1": [ 91, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 100, 33 ], "axis1": [ 94, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 100, 33 ], "axis1": [ 99, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 100, 33 ], "axis1": [ 335, 8 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 100, 33 ], "axis1": [ 344, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 100, 33 ], "axis1": [ 439, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 100, 33 ], "axis1": [ 480, 16 ], "matrix": "L15.mlp.W_out" } ], "l:377": [ { "axis0": [ 0, 1 ], "axis1": [ 702, 82 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 702, 82 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 247, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1060, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1062, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1064, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1226, 12 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1239, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1244, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1249, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1254, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1824, 1 ], "axis1": [ 702, 82 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 702, 82 ], "axis1": [ 247, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 702, 82 ], "axis1": [ 1060, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 702, 82 ], "axis1": [ 1062, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 702, 82 ], "axis1": [ 1064, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 702, 82 ], "axis1": [ 1226, 12 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 702, 82 ], "axis1": [ 1239, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 702, 82 ], "axis1": [ 1244, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 702, 82 ], "axis1": [ 1249, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 702, 82 ], "axis1": [ 1254, 3 ], "matrix": "L13.mlp.W_out" } ], "l:378": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 212, 1 ], "axis1": [ 0, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 1824, 1 ], "matrix": "L13.attn.W_O" } ], "l:379": [ { "axis0": [ 0, 1 ], "axis1": [ 54, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1559, 33 ], "axis1": [ 54, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1559, 33 ], "axis1": [ 55, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 54, 1 ], "axis1": [ 1824, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 55, 1 ], "axis1": [ 1824, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:380": [ { "axis0": [ 153, 1 ], "axis1": [ 1536, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 153, 1 ], "axis1": [ 1536, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1536, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1536, 64 ], "axis1": [ 1559, 33 ], "matrix": "L1.attn.W_O" } ], "l:381": [ { "axis0": [ 0, 1 ], "axis1": [ 153, 1 ], "matrix": "L0.mlp.W_out" } ], "l:382": [ { "axis0": [ 0, 1 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 676, 8 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 869, 8 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 878, 1 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 883, 5 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 930, 1 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 965, 10 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 676, 8 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 869, 8 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 878, 1 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 883, 5 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 930, 1 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 965, 10 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 133, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 133, 33 ], "axis1": [ 101, 5 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 133, 33 ], "axis1": [ 514, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 133, 33 ], "axis1": [ 669, 6 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 133, 33 ], "axis1": [ 868, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 133, 33 ], "axis1": [ 881, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 133, 33 ], "axis1": [ 959, 6 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 133, 33 ], "axis1": [ 982, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 133, 33 ], "axis1": [ 986, 10 ], "matrix": "L15.mlp.W_out" } ], "l:384": [ { "axis0": [ 0, 1 ], "axis1": [ 370, 9 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 370, 9 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 291, 1 ], "axis1": [ 370, 9 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 370, 9 ], "axis1": [ 676, 8 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 370, 9 ], "axis1": [ 869, 8 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 370, 9 ], "axis1": [ 878, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 370, 9 ], "axis1": [ 883, 5 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 370, 9 ], "axis1": [ 930, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 370, 9 ], "axis1": [ 965, 10 ], "matrix": "L14.mlp.W_out" } ], "l:385": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 1834, 1 ], "axis1": [ 128, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 291, 1 ], "matrix": "L14.attn.W_O" } ], "l:386": [ { "axis0": [ 0, 1 ], "axis1": [ 981, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 72, 1 ], "axis1": [ 981, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 72, 1 ], "axis1": [ 982, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 981, 1 ], "axis1": [ 291, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 982, 1 ], "axis1": [ 291, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" } ], "l:387": [ { "axis0": [ 0, 1 ], "axis1": [ 13, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 13, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 244, 1 ], "axis1": [ 13, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 959, 1 ], "axis1": [ 13, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 989, 1 ], "axis1": [ 13, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 244, 1 ], "axis1": [ 13, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 959, 1 ], "axis1": [ 13, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 989, 1 ], "axis1": [ 13, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 13, 2 ], "axis1": [ 291, 1 ], "matrix": "L12.mlp.W_out" } ], "l:389": [ { "axis0": [ 0, 1 ], "axis1": [ 86, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 86, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 989, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1814, 1 ], "axis1": [ 86, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 86, 2 ], "axis1": [ 989, 1 ], "matrix": "L2.mlp.W_out" } ], "l:39": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1796, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 983, 2 ], "axis1": [ 1, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 1796, 1 ], "matrix": "L3.mlp.W_out" } ], "l:390": [ { "axis0": [ 0, 1 ], "axis1": [ 34, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 866, 33 ], "axis1": [ 34, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 866, 33 ], "axis1": [ 35, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 34, 1 ], "axis1": [ 1814, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 35, 1 ], "axis1": [ 1814, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:391": [ { "axis0": [ 121, 1 ], "axis1": [ 192, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 121, 1 ], "axis1": [ 192, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 192, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 866, 33 ], "matrix": "L1.attn.W_O" } ], "l:392": [ { "axis0": [ 0, 1 ], "axis1": [ 121, 1 ], "matrix": "L0.mlp.W_out" } ], "l:393": [ { "axis0": [ 0, 1 ], "axis1": [ 308, 2 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 35, 6 ], "axis1": [ 308, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 42, 3 ], "axis1": [ 308, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 46, 9 ], "axis1": [ 308, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 308, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 61, 3 ], "axis1": [ 308, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 65, 11 ], "axis1": [ 308, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 35, 6 ], "axis1": [ 309, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 42, 3 ], "axis1": [ 309, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 46, 9 ], "axis1": [ 309, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 309, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 61, 3 ], "axis1": [ 309, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 65, 11 ], "axis1": [ 309, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 308, 1 ], "axis1": [ 244, 1 ], "diagonal": true, "matrix": "L9.mlp.W_out" }, { "axis0": [ 309, 1 ], "axis1": [ 244, 1 ], "diagonal": true, "matrix": "L9.mlp.W_out" } ], "l:394": [ { "axis0": [ 0, 1 ], "axis1": [ 1867, 2 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 34, 1 ], "axis1": [ 1867, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 1867, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 172, 2 ], "axis1": [ 1867, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 196, 1 ], "axis1": [ 1867, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 199, 1 ], "axis1": [ 1867, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 223, 12 ], "axis1": [ 1867, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 254, 13 ], "axis1": [ 1867, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 1867, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1150, 1 ], "axis1": [ 1867, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 34, 1 ], "axis1": [ 1868, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 1868, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 172, 2 ], "axis1": [ 1868, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 196, 1 ], "axis1": [ 1868, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 199, 1 ], "axis1": [ 1868, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 223, 12 ], "axis1": [ 1868, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 254, 13 ], "axis1": [ 1868, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 1868, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1150, 1 ], "axis1": [ 1868, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1867, 1 ], "axis1": [ 959, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1868, 1 ], "axis1": [ 959, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" } ], "l:395": [ { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 286, 1 ], "axis1": [ 512, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 72, 1 ], "matrix": "L13.attn.W_O" } ], "l:396": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 244, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 959, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 989, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 244, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 959, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 989, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 3, 2 ], "axis1": [ 286, 1 ], "matrix": "L12.mlp.W_out" } ], "l:398": [ { "axis0": [ 0, 1 ], "axis1": [ 74, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 58, 1 ], "axis1": [ 74, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 75, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 74, 1 ], "axis1": [ 1834, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 75, 1 ], "axis1": [ 1834, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:399": [ { "axis0": [ 0, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 47, 2 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 172, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 237, 4 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 244, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 335, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 881, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1019, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1049, 2 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1149, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1170, 9 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1184, 6 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1223, 3 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 47, 2 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 172, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 237, 4 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 244, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 335, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 881, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1019, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1049, 2 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1149, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1170, 9 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1184, 6 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1223, 3 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 557, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 557, 33 ], "axis1": [ 1749, 33 ], "matrix": "L14.mlp.W_out" } ], "l:4": [ { "axis0": [ 0, 1 ], "axis1": [ 828, 1 ], "matrix": "L0.mlp.W_out" } ], "l:401": [ { "axis0": [ 0, 1 ], "axis1": [ 696, 6 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 696, 6 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 42, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 47, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 172, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 237, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 244, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 335, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 881, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1019, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1049, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1149, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1170, 9 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1184, 6 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1223, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 82, 1 ], "axis1": [ 696, 6 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 696, 6 ], "axis1": [ 42, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 696, 6 ], "axis1": [ 47, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 696, 6 ], "axis1": [ 172, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 696, 6 ], "axis1": [ 237, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 696, 6 ], "axis1": [ 244, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 696, 6 ], "axis1": [ 335, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 696, 6 ], "axis1": [ 881, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 696, 6 ], "axis1": [ 1019, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 696, 6 ], "axis1": [ 1049, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 696, 6 ], "axis1": [ 1149, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 696, 6 ], "axis1": [ 1170, 9 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 696, 6 ], "axis1": [ 1184, 6 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 696, 6 ], "axis1": [ 1223, 3 ], "matrix": "L13.mlp.W_out" } ], "l:402": [ { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 60, 1 ], "axis1": [ 1152, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1088, 1 ], "axis1": [ 1152, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1152, 64 ], "axis1": [ 82, 1 ], "matrix": "L13.attn.W_O" } ], "l:403": [ { "axis0": [ 0, 1 ], "axis1": [ 202, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 339, 2 ], "axis1": [ 202, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 339, 2 ], "axis1": [ 203, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 202, 1 ], "axis1": [ 1088, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 203, 1 ], "axis1": [ 1088, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" } ], "l:404": [ { "axis0": [ 0, 1 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 133, 7 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 143, 9 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 158, 7 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 168, 1 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 171, 4 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 177, 5 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1063, 1 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 133, 7 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 143, 9 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 158, 7 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 168, 1 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 171, 4 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 177, 5 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1063, 1 ], "axis1": [ 133, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 133, 33 ], "axis1": [ 407, 33 ], "matrix": "L16.mlp.W_out" } ], "l:406": [ { "axis0": [ 0, 1 ], "axis1": [ 886, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 886, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1063, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1028, 2 ], "axis1": [ 886, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 886, 2 ], "axis1": [ 1063, 1 ], "matrix": "L4.mlp.W_out" } ], "l:408": [ { "axis0": [ 0, 1 ], "axis1": [ 192, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1028, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 192, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 192, 2 ], "axis1": [ 1028, 1 ], "matrix": "L3.mlp.W_out" } ], "l:409": [ { "axis0": [ 0, 1 ], "axis1": [ 194, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 194, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 194, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1029, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 194, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 194, 2 ], "axis1": [ 1029, 1 ], "matrix": "L3.mlp.W_out" } ], "l:41": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 983, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1804, 3 ], "axis1": [ 1, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 983, 1 ], "matrix": "L2.mlp.W_out" } ], "l:410": [ { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 996, 3 ], "axis1": [ 384, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1000, 6 ], "axis1": [ 384, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1010, 10 ], "axis1": [ 384, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1023, 1 ], "axis1": [ 384, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1025, 1 ], "axis1": [ 384, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1027, 1 ], "axis1": [ 384, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1031, 7 ], "axis1": [ 384, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1042, 1 ], "axis1": [ 384, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1049, 1 ], "axis1": [ 384, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1782, 8 ], "axis1": [ 384, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1791, 25 ], "axis1": [ 384, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 133, 7 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 143, 9 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 158, 7 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 168, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 171, 4 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 177, 5 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 1050, 1 ], "axis1": [ 448, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1065, 3 ], "axis1": [ 448, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1089, 8 ], "axis1": [ 448, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1111, 1 ], "axis1": [ 448, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1131, 15 ], "axis1": [ 448, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1147, 2 ], "axis1": [ 448, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1150, 1 ], "axis1": [ 448, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1152, 3 ], "axis1": [ 448, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1855, 1 ], "axis1": [ 448, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 133, 7 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 143, 9 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 158, 7 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 168, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 171, 4 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 177, 5 ], "matrix": "L16.attn.W_O" } ], "l:412": [ { "axis0": [ 0, 1 ], "axis1": [ 590, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 590, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1131, 15 ], "axis1": [ 590, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1152, 18 ], "axis1": [ 590, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 590, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1131, 15 ], "axis1": [ 590, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1152, 18 ], "axis1": [ 590, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 590, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 590, 33 ], "axis1": [ 1782, 8 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 590, 33 ], "axis1": [ 1791, 25 ], "matrix": "L14.mlp.W_out" } ], "l:414": [ { "axis0": [ 0, 1 ], "axis1": [ 614, 82 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 614, 82 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1131, 15 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 18 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1825, 1 ], "axis1": [ 614, 82 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 614, 82 ], "axis1": [ 1131, 15 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 614, 82 ], "axis1": [ 1152, 18 ], "matrix": "L13.mlp.W_out" } ], "l:415": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 1243, 1 ], "axis1": [ 64, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 1825, 1 ], "matrix": "L13.attn.W_O" } ], "l:416": [ { "axis0": [ 0, 1 ], "axis1": [ 56, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1592, 33 ], "axis1": [ 56, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1592, 33 ], "axis1": [ 57, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 56, 1 ], "axis1": [ 1825, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 57, 1 ], "axis1": [ 1825, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:417": [ { "axis0": [ 154, 1 ], "axis1": [ 1600, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 154, 1 ], "axis1": [ 1600, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1600, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1600, 64 ], "axis1": [ 1592, 33 ], "matrix": "L1.attn.W_O" } ], "l:418": [ { "axis0": [ 0, 1 ], "axis1": [ 154, 1 ], "matrix": "L0.mlp.W_out" } ], "l:419": [ { "axis0": [ 0, 1 ], "axis1": [ 166, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 166, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 517, 32 ], "axis1": [ 166, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 675, 1 ], "axis1": [ 166, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 166, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 517, 32 ], "axis1": [ 166, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 675, 1 ], "axis1": [ 166, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 166, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 166, 33 ], "axis1": [ 996, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 166, 33 ], "axis1": [ 1000, 6 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 166, 33 ], "axis1": [ 1010, 10 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 166, 33 ], "axis1": [ 1023, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 166, 33 ], "axis1": [ 1025, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 166, 33 ], "axis1": [ 1027, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 166, 33 ], "axis1": [ 1031, 7 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 166, 33 ], "axis1": [ 1042, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 166, 33 ], "axis1": [ 1049, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 166, 33 ], "axis1": [ 1066, 1 ], "matrix": "L15.mlp.W_out" } ], "l:421": [ { "axis0": [ 0, 1 ], "axis1": [ 289, 81 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 289, 81 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 58, 1 ], "axis1": [ 289, 81 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 289, 81 ], "axis1": [ 517, 32 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 289, 81 ], "axis1": [ 675, 1 ], "matrix": "L14.mlp.W_out" } ], "l:422": [ { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 1588, 1 ], "axis1": [ 192, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 58, 1 ], "matrix": "L14.attn.W_O" } ], "l:423": [ { "axis0": [ 0, 1 ], "axis1": [ 983, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 73, 1 ], "axis1": [ 983, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 73, 1 ], "axis1": [ 984, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 983, 1 ], "axis1": [ 58, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 984, 1 ], "axis1": [ 58, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" } ], "l:424": [ { "axis0": [ 0, 1 ], "axis1": [ 15, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 15, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 244, 1 ], "axis1": [ 15, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 959, 1 ], "axis1": [ 15, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 990, 1 ], "axis1": [ 15, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 244, 1 ], "axis1": [ 15, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 959, 1 ], "axis1": [ 15, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 990, 1 ], "axis1": [ 15, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 15, 2 ], "axis1": [ 58, 1 ], "matrix": "L12.mlp.W_out" } ], "l:426": [ { "axis0": [ 0, 1 ], "axis1": [ 88, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 88, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 990, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1815, 1 ], "axis1": [ 88, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 88, 2 ], "axis1": [ 990, 1 ], "matrix": "L2.mlp.W_out" } ], "l:427": [ { "axis0": [ 0, 1 ], "axis1": [ 36, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 899, 33 ], "axis1": [ 36, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 899, 33 ], "axis1": [ 37, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 36, 1 ], "axis1": [ 1815, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 37, 1 ], "axis1": [ 1815, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:428": [ { "axis0": [ 122, 1 ], "axis1": [ 256, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 122, 1 ], "axis1": [ 256, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 256, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 899, 33 ], "matrix": "L1.attn.W_O" } ], "l:429": [ { "axis0": [ 0, 1 ], "axis1": [ 122, 1 ], "matrix": "L0.mlp.W_out" } ], "l:43": [ { "axis0": [ 0, 1 ], "axis1": [ 14, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 14, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 38, 1 ], "axis1": [ 14, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 14, 2 ], "axis1": [ 1804, 1 ], "matrix": "L1.mlp.W_out" } ], "l:430": [ { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 287, 1 ], "axis1": [ 576, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 73, 1 ], "matrix": "L13.attn.W_O" } ], "l:431": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 244, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 959, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 990, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 244, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 959, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 990, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 5, 2 ], "axis1": [ 287, 1 ], "matrix": "L12.mlp.W_out" } ], "l:433": [ { "axis0": [ 0, 1 ], "axis1": [ 975, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 1822, 1 ], "axis1": [ 975, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1822, 1 ], "axis1": [ 976, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 975, 1 ], "axis1": [ 1588, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 976, 1 ], "axis1": [ 1588, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" } ], "l:434": [ { "axis0": [ 0, 1 ], "axis1": [ 50, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1493, 33 ], "axis1": [ 50, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1493, 33 ], "axis1": [ 51, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 50, 1 ], "axis1": [ 1822, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 51, 1 ], "axis1": [ 1822, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:435": [ { "axis0": [ 151, 1 ], "axis1": [ 1408, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 151, 1 ], "axis1": [ 1408, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1408, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1408, 64 ], "axis1": [ 1493, 33 ], "matrix": "L1.attn.W_O" } ], "l:436": [ { "axis0": [ 0, 1 ], "axis1": [ 151, 1 ], "matrix": "L0.mlp.W_out" } ], "l:437": [ { "axis0": [ 0, 1 ], "axis1": [ 199, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 199, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1097, 14 ], "axis1": [ 199, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1112, 19 ], "axis1": [ 199, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 199, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1097, 14 ], "axis1": [ 199, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1112, 19 ], "axis1": [ 199, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 199, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 199, 33 ], "axis1": [ 1065, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 199, 33 ], "axis1": [ 1067, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 199, 33 ], "axis1": [ 1089, 8 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 199, 33 ], "axis1": [ 1111, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 199, 33 ], "axis1": [ 1131, 15 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 199, 33 ], "axis1": [ 1147, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 199, 33 ], "axis1": [ 1150, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 199, 33 ], "axis1": [ 1152, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 199, 33 ], "axis1": [ 1855, 1 ], "matrix": "L15.mlp.W_out" } ], "l:439": [ { "axis0": [ 0, 1 ], "axis1": [ 552, 62 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 552, 62 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1097, 14 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1112, 19 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1826, 1 ], "axis1": [ 552, 62 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 552, 62 ], "axis1": [ 1097, 14 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 552, 62 ], "axis1": [ 1112, 19 ], "matrix": "L13.mlp.W_out" } ], "l:44": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 38, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 33 ], "axis1": [ 1, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 1, 1 ], "axis1": [ 38, 1 ], "matrix": "L0.mlp.W_out" } ], "l:440": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 1088, 1 ], "axis1": [ 128, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 1826, 1 ], "matrix": "L13.attn.W_O" } ], "l:441": [ { "axis0": [ 0, 1 ], "axis1": [ 58, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1625, 33 ], "axis1": [ 58, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1625, 33 ], "axis1": [ 59, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 1826, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 59, 1 ], "axis1": [ 1826, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:442": [ { "axis0": [ 155, 1 ], "axis1": [ 1664, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 155, 1 ], "axis1": [ 1664, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1664, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1664, 64 ], "axis1": [ 1625, 33 ], "matrix": "L1.attn.W_O" } ], "l:443": [ { "axis0": [ 0, 1 ], "axis1": [ 155, 1 ], "matrix": "L0.mlp.W_out" } ], "l:444": [ { "axis0": [ 0, 1 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 182, 11 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 195, 7 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 203, 1 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 212, 1 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 222, 13 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1558, 1 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 182, 11 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 195, 7 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 203, 1 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 212, 1 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 222, 13 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1558, 1 ], "axis1": [ 166, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 166, 33 ], "axis1": [ 440, 33 ], "matrix": "L16.mlp.W_out" } ], "l:446": [ { "axis0": [ 0, 1 ], "axis1": [ 888, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 888, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1558, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1030, 2 ], "axis1": [ 888, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 888, 2 ], "axis1": [ 1558, 1 ], "matrix": "L4.mlp.W_out" } ], "l:448": [ { "axis0": [ 0, 1 ], "axis1": [ 196, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 196, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 196, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1030, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 196, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 196, 2 ], "axis1": [ 1030, 1 ], "matrix": "L3.mlp.W_out" } ], "l:449": [ { "axis0": [ 0, 1 ], "axis1": [ 198, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 198, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 198, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1031, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 198, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 198, 2 ], "axis1": [ 1031, 1 ], "matrix": "L3.mlp.W_out" } ], "l:45": [ { "axis0": [ 0, 1 ], "axis1": [ 16, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 16, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 39, 1 ], "axis1": [ 16, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 16, 2 ], "axis1": [ 1805, 1 ], "matrix": "L1.mlp.W_out" } ], "l:450": [ { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 1155, 24 ], "axis1": [ 512, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1184, 6 ], "axis1": [ 512, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1223, 1 ], "axis1": [ 512, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1520, 15 ], "axis1": [ 512, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1536, 18 ], "axis1": [ 512, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 182, 11 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 195, 7 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 203, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 212, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 222, 13 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 251, 3 ], "axis1": [ 576, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1224, 2 ], "axis1": [ 576, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1238, 1 ], "axis1": [ 576, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1247, 2 ], "axis1": [ 576, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1253, 1 ], "axis1": [ 576, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1290, 12 ], "axis1": [ 576, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1303, 1 ], "axis1": [ 576, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1305, 1 ], "axis1": [ 576, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1876, 1 ], "axis1": [ 576, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1878, 11 ], "axis1": [ 576, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 182, 11 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 195, 7 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 203, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 212, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 222, 13 ], "matrix": "L16.attn.W_O" } ], "l:452": [ { "axis0": [ 0, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 37, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 109, 6 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 118, 2 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 186, 2 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 232, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 496, 18 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 515, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 930, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 37, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 69, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 109, 6 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 118, 2 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 186, 2 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 232, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 496, 18 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 515, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 930, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 909, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 909, 33 ], "axis1": [ 1520, 15 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 909, 33 ], "axis1": [ 1536, 18 ], "matrix": "L13.mlp.W_out" } ], "l:454": [ { "axis0": [ 0, 1 ], "axis1": [ 57, 36 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 57, 36 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 37, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 109, 6 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 118, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 186, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 232, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 496, 18 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 515, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 930, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 201, 1 ], "axis1": [ 57, 36 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 57, 36 ], "axis1": [ 37, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 57, 36 ], "axis1": [ 69, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 57, 36 ], "axis1": [ 109, 6 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 57, 36 ], "axis1": [ 118, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 57, 36 ], "axis1": [ 186, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 57, 36 ], "axis1": [ 232, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 57, 36 ], "axis1": [ 496, 18 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 57, 36 ], "axis1": [ 515, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 57, 36 ], "axis1": [ 930, 1 ], "matrix": "L12.mlp.W_out" } ], "l:455": [ { "axis0": [ 0, 1 ], "axis1": [ 232, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 232, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 467, 13 ], "axis1": [ 232, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 496, 18 ], "axis1": [ 232, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 515, 2 ], "axis1": [ 232, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 232, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 467, 13 ], "axis1": [ 232, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 496, 18 ], "axis1": [ 232, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 515, 2 ], "axis1": [ 232, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 232, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 232, 33 ], "axis1": [ 1155, 24 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 232, 33 ], "axis1": [ 1184, 6 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 232, 33 ], "axis1": [ 1223, 3 ], "matrix": "L15.mlp.W_out" } ], "l:457": [ { "axis0": [ 0, 1 ], "axis1": [ 208, 81 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 208, 81 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 293, 1 ], "axis1": [ 208, 81 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 208, 81 ], "axis1": [ 467, 13 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 208, 81 ], "axis1": [ 496, 18 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 208, 81 ], "axis1": [ 515, 2 ], "matrix": "L14.mlp.W_out" } ], "l:458": [ { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 982, 1 ], "axis1": [ 256, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 293, 1 ], "matrix": "L14.attn.W_O" } ], "l:459": [ { "axis0": [ 0, 1 ], "axis1": [ 985, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 74, 1 ], "axis1": [ 985, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 74, 1 ], "axis1": [ 986, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 985, 1 ], "axis1": [ 293, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 986, 1 ], "axis1": [ 293, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" } ], "l:46": [ { "axis0": [ 0, 1 ], "axis1": [ 2, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 2, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 39, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 33 ], "axis1": [ 2, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 2, 1 ], "axis1": [ 39, 1 ], "matrix": "L0.mlp.W_out" } ], "l:460": [ { "axis0": [ 0, 1 ], "axis1": [ 17, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 17, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 246, 1 ], "axis1": [ 17, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 960, 1 ], "axis1": [ 17, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 991, 1 ], "axis1": [ 17, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 246, 1 ], "axis1": [ 17, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 960, 1 ], "axis1": [ 17, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 991, 1 ], "axis1": [ 17, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 17, 2 ], "axis1": [ 293, 1 ], "matrix": "L12.mlp.W_out" } ], "l:462": [ { "axis0": [ 0, 1 ], "axis1": [ 90, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 90, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 991, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1816, 1 ], "axis1": [ 90, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 90, 2 ], "axis1": [ 991, 1 ], "matrix": "L2.mlp.W_out" } ], "l:463": [ { "axis0": [ 0, 1 ], "axis1": [ 38, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 932, 33 ], "axis1": [ 38, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 932, 33 ], "axis1": [ 39, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 38, 1 ], "axis1": [ 1816, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 39, 1 ], "axis1": [ 1816, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:464": [ { "axis0": [ 123, 1 ], "axis1": [ 320, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 123, 1 ], "axis1": [ 320, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 320, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 932, 33 ], "matrix": "L1.attn.W_O" } ], "l:465": [ { "axis0": [ 0, 1 ], "axis1": [ 123, 1 ], "matrix": "L0.mlp.W_out" } ], "l:466": [ { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 288, 1 ], "axis1": [ 640, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 74, 1 ], "matrix": "L13.attn.W_O" } ], "l:467": [ { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 246, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 960, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 991, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 246, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 960, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 991, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 7, 2 ], "axis1": [ 288, 1 ], "matrix": "L12.mlp.W_out" } ], "l:469": [ { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1823, 1 ], "axis1": [ 960, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 960, 64 ], "axis1": [ 982, 1 ], "matrix": "L2.attn.W_O" } ], "l:47": [ { "axis0": [ 0, 1 ], "axis1": [ 18, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 18, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 18, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 18, 2 ], "axis1": [ 1806, 1 ], "matrix": "L1.mlp.W_out" } ], "l:470": [ { "axis0": [ 0, 1 ], "axis1": [ 52, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1526, 33 ], "axis1": [ 52, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1526, 33 ], "axis1": [ 53, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 1823, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 53, 1 ], "axis1": [ 1823, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:471": [ { "axis0": [ 152, 1 ], "axis1": [ 1472, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 152, 1 ], "axis1": [ 1472, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1472, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1472, 64 ], "axis1": [ 1526, 33 ], "matrix": "L1.attn.W_O" } ], "l:472": [ { "axis0": [ 0, 1 ], "axis1": [ 152, 1 ], "matrix": "L0.mlp.W_out" } ], "l:473": [ { "axis0": [ 0, 1 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 324, 5 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 330, 2 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 333, 2 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 346, 1 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 444, 23 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 324, 5 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 330, 2 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 333, 2 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 346, 1 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 444, 23 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 265, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 265, 33 ], "axis1": [ 251, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 265, 33 ], "axis1": [ 1238, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 265, 33 ], "axis1": [ 1247, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 265, 33 ], "axis1": [ 1253, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 265, 33 ], "axis1": [ 1290, 12 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 265, 33 ], "axis1": [ 1303, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 265, 33 ], "axis1": [ 1305, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 265, 33 ], "axis1": [ 1876, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 265, 33 ], "axis1": [ 1878, 11 ], "matrix": "L15.mlp.W_out" } ], "l:475": [ { "axis0": [ 0, 1 ], "axis1": [ 74, 134 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 74, 134 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 324, 5 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 330, 2 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 333, 2 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 346, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 444, 23 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1827, 1 ], "axis1": [ 74, 134 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 74, 134 ], "axis1": [ 324, 5 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 74, 134 ], "axis1": [ 330, 2 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 74, 134 ], "axis1": [ 333, 2 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 74, 134 ], "axis1": [ 346, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 74, 134 ], "axis1": [ 444, 23 ], "matrix": "L14.mlp.W_out" } ], "l:476": [ { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 1030, 1 ], "axis1": [ 192, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 1827, 1 ], "matrix": "L13.attn.W_O" } ], "l:477": [ { "axis0": [ 0, 1 ], "axis1": [ 60, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1658, 33 ], "axis1": [ 60, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1658, 33 ], "axis1": [ 61, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 60, 1 ], "axis1": [ 1827, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 61, 1 ], "axis1": [ 1827, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:478": [ { "axis0": [ 156, 1 ], "axis1": [ 1728, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 156, 1 ], "axis1": [ 1728, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1728, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1728, 64 ], "axis1": [ 1658, 33 ], "matrix": "L1.attn.W_O" } ], "l:479": [ { "axis0": [ 0, 1 ], "axis1": [ 156, 1 ], "matrix": "L0.mlp.W_out" } ], "l:48": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 40, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 33 ], "axis1": [ 3, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 3, 1 ], "axis1": [ 40, 1 ], "matrix": "L0.mlp.W_out" } ], "l:480": [ { "axis0": [ 0, 1 ], "axis1": [ 199, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 199, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 237, 5 ], "axis1": [ 199, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 244, 28 ], "axis1": [ 199, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1432, 1 ], "axis1": [ 199, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 237, 5 ], "axis1": [ 199, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 244, 28 ], "axis1": [ 199, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1432, 1 ], "axis1": [ 199, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 199, 33 ], "axis1": [ 473, 33 ], "matrix": "L16.mlp.W_out" } ], "l:482": [ { "axis0": [ 0, 1 ], "axis1": [ 890, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 890, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1432, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1032, 2 ], "axis1": [ 890, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 890, 2 ], "axis1": [ 1432, 1 ], "matrix": "L4.mlp.W_out" } ], "l:484": [ { "axis0": [ 0, 1 ], "axis1": [ 200, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 200, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 200, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1032, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 200, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 200, 2 ], "axis1": [ 1032, 1 ], "matrix": "L3.mlp.W_out" } ], "l:485": [ { "axis0": [ 0, 1 ], "axis1": [ 202, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 202, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 202, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1033, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 202, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 202, 2 ], "axis1": [ 1033, 1 ], "matrix": "L3.mlp.W_out" } ], "l:486": [ { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 1306, 8 ], "axis1": [ 640, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1332, 1 ], "axis1": [ 640, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1347, 52 ], "axis1": [ 640, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1682, 1 ], "axis1": [ 640, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1704, 1 ], "axis1": [ 640, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1903, 1 ], "axis1": [ 640, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 237, 5 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 244, 28 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 1816, 11 ], "axis1": [ 704, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1828, 3 ], "axis1": [ 704, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1832, 1 ], "axis1": [ 704, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1835, 6 ], "axis1": [ 704, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1842, 12 ], "axis1": [ 704, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1904, 2 ], "axis1": [ 704, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 237, 5 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 244, 28 ], "matrix": "L16.attn.W_O" } ], "l:488": [ { "axis0": [ 0, 1 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 921, 9 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 931, 4 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 936, 4 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 941, 4 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 946, 4 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 951, 3 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1813, 5 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 921, 9 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 931, 4 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 936, 4 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 941, 4 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 946, 4 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 951, 3 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1813, 5 ], "axis1": [ 346, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 346, 33 ], "axis1": [ 1347, 10 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 346, 33 ], "axis1": [ 1358, 11 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 346, 33 ], "axis1": [ 1370, 10 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 346, 33 ], "axis1": [ 1682, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 346, 33 ], "axis1": [ 1704, 1 ], "matrix": "L4.mlp.W_out" } ], "l:49": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 984, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1662, 1 ], "axis1": [ 3, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 3, 2 ], "axis1": [ 984, 1 ], "matrix": "L2.mlp.W_out" } ], "l:490": [ { "axis0": [ 0, 1 ], "axis1": [ 133, 5 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 133, 5 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 921, 9 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 931, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 936, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 941, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 946, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 951, 3 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1813, 5 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 1009, 1 ], "axis1": [ 133, 5 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 133, 5 ], "axis1": [ 921, 9 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 133, 5 ], "axis1": [ 931, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 133, 5 ], "axis1": [ 936, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 133, 5 ], "axis1": [ 941, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 133, 5 ], "axis1": [ 946, 4 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 133, 5 ], "axis1": [ 951, 3 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 133, 5 ], "axis1": [ 1813, 5 ], "matrix": "L3.mlp.W_out" } ], "l:491": [ { "axis0": [ 0, 1 ], "axis1": [ 141, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 141, 1 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1009, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 914, 33 ], "axis1": [ 141, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1009, 1 ], "matrix": "L2.mlp.W_out" } ], "l:492": [ { "axis0": [ 1843, 1 ], "axis1": [ 704, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 1843, 1 ], "axis1": [ 704, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 704, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 914, 33 ], "matrix": "L2.attn.W_O" } ], "l:493": [ { "axis0": [ 0, 1 ], "axis1": [ 1843, 1 ], "matrix": "L1.mlp.W_out" } ], "l:494": [ { "axis0": [ 0, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 188, 4 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 212, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 228, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 232, 3 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 241, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 294, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 297, 10 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 313, 4 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 318, 3 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 322, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 188, 4 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 212, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 228, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 232, 3 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 241, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 294, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 297, 10 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 308, 4 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 313, 4 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 318, 3 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 322, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 298, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 298, 33 ], "axis1": [ 1306, 8 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 298, 33 ], "axis1": [ 1332, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 298, 33 ], "axis1": [ 1357, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 298, 33 ], "axis1": [ 1369, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 298, 33 ], "axis1": [ 1380, 19 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 298, 33 ], "axis1": [ 1903, 3 ], "matrix": "L15.mlp.W_out" } ], "l:496": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 73 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 73 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 188, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 212, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 228, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 232, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 241, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 294, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 297, 10 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 308, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 313, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 318, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 322, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 296, 1 ], "axis1": [ 1, 73 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1, 73 ], "axis1": [ 188, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 73 ], "axis1": [ 212, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 73 ], "axis1": [ 228, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 73 ], "axis1": [ 232, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 73 ], "axis1": [ 241, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 73 ], "axis1": [ 294, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 73 ], "axis1": [ 297, 10 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 73 ], "axis1": [ 308, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 73 ], "axis1": [ 313, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 73 ], "axis1": [ 318, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 73 ], "axis1": [ 322, 1 ], "matrix": "L14.mlp.W_out" } ], "l:497": [ { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 1589, 1 ], "axis1": [ 320, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 296, 1 ], "matrix": "L14.attn.W_O" } ], "l:498": [ { "axis0": [ 0, 1 ], "axis1": [ 23, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 23, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 245, 1 ], "axis1": [ 23, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 961, 1 ], "axis1": [ 23, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 992, 1 ], "axis1": [ 23, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 245, 1 ], "axis1": [ 23, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 961, 1 ], "axis1": [ 23, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 992, 1 ], "axis1": [ 23, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 23, 2 ], "axis1": [ 296, 1 ], "matrix": "L12.mlp.W_out" } ], "l:5": [ { "axis0": [ 0, 1 ], "axis1": [ 829, 1 ], "matrix": "L0.mlp.W_out" } ], "l:50": [ { "axis0": [ 1835, 2 ], "axis1": [ 0, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 41, 1 ], "axis1": [ 0, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1837, 1 ], "axis1": [ 0, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 41, 1 ], "axis1": [ 0, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 1662, 1 ], "matrix": "L2.attn.W_O" } ], "l:500": [ { "axis0": [ 0, 1 ], "axis1": [ 92, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 92, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 992, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1818, 1 ], "axis1": [ 92, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 92, 2 ], "axis1": [ 992, 1 ], "matrix": "L2.mlp.W_out" } ], "l:501": [ { "axis0": [ 0, 1 ], "axis1": [ 42, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1064, 33 ], "axis1": [ 42, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1064, 33 ], "axis1": [ 43, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 1818, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 43, 1 ], "axis1": [ 1818, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:502": [ { "axis0": [ 128, 1 ], "axis1": [ 576, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 128, 1 ], "axis1": [ 576, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 576, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 1064, 33 ], "matrix": "L1.attn.W_O" } ], "l:503": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 1 ], "matrix": "L0.mlp.W_out" } ], "l:504": [ { "axis0": [ 0, 1 ], "axis1": [ 977, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 294, 1 ], "axis1": [ 977, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 294, 1 ], "axis1": [ 978, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 977, 1 ], "axis1": [ 1589, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 978, 1 ], "axis1": [ 1589, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" } ], "l:505": [ { "axis0": [ 0, 1 ], "axis1": [ 19, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 19, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 245, 1 ], "axis1": [ 19, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 961, 1 ], "axis1": [ 19, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 992, 1 ], "axis1": [ 19, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 245, 1 ], "axis1": [ 19, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 961, 1 ], "axis1": [ 19, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 992, 1 ], "axis1": [ 19, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 19, 2 ], "axis1": [ 294, 1 ], "matrix": "L12.mlp.W_out" } ], "l:507": [ { "axis0": [ 0, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 39, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 43, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 177, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 192, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 195, 2 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 199, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 201, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 670, 5 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 959, 6 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 988, 5 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1031, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1089, 3 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1093, 4 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 39, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 41, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 43, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 177, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 192, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 195, 2 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 199, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 201, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 670, 5 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 959, 6 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 988, 5 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1031, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1089, 3 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1093, 4 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 623, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 623, 33 ], "axis1": [ 1816, 11 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 623, 33 ], "axis1": [ 1828, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 623, 33 ], "axis1": [ 1832, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 623, 33 ], "axis1": [ 1835, 6 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 623, 33 ], "axis1": [ 1842, 12 ], "matrix": "L14.mlp.W_out" } ], "l:509": [ { "axis0": [ 0, 1 ], "axis1": [ 346, 206 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 346, 206 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 39, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 41, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 43, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 177, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 195, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 199, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 201, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 670, 5 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 959, 6 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 988, 5 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1031, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1089, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1093, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1828, 1 ], "axis1": [ 346, 206 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 346, 206 ], "axis1": [ 39, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 206 ], "axis1": [ 41, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 206 ], "axis1": [ 43, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 206 ], "axis1": [ 177, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 206 ], "axis1": [ 192, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 206 ], "axis1": [ 195, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 206 ], "axis1": [ 199, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 206 ], "axis1": [ 201, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 206 ], "axis1": [ 670, 5 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 206 ], "axis1": [ 959, 6 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 206 ], "axis1": [ 988, 5 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 206 ], "axis1": [ 1031, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 206 ], "axis1": [ 1089, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 206 ], "axis1": [ 1093, 4 ], "matrix": "L13.mlp.W_out" } ], "l:510": [ { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 1086, 1 ], "axis1": [ 256, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 1828, 1 ], "matrix": "L13.attn.W_O" } ], "l:511": [ { "axis0": [ 0, 1 ], "axis1": [ 62, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1691, 33 ], "axis1": [ 62, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1691, 33 ], "axis1": [ 63, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 62, 1 ], "axis1": [ 1828, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 63, 1 ], "axis1": [ 1828, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:512": [ { "axis0": [ 157, 1 ], "axis1": [ 1792, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 157, 1 ], "axis1": [ 1792, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1792, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1792, 64 ], "axis1": [ 1691, 33 ], "matrix": "L1.attn.W_O" } ], "l:513": [ { "axis0": [ 0, 1 ], "axis1": [ 157, 1 ], "matrix": "L0.mlp.W_out" } ], "l:514": [ { "axis0": [ 0, 1 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 67, 2 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 70, 13 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 84, 7 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 96, 3 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 130, 3 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 143, 3 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1535, 1 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1643, 1 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1857, 1 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 67, 2 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 70, 13 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 84, 7 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 96, 3 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 130, 3 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 143, 3 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1535, 1 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1643, 1 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1857, 1 ], "axis1": [ 366, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 366, 33 ], "axis1": [ 307, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 366, 33 ], "axis1": [ 312, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 366, 33 ], "axis1": [ 317, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 366, 33 ], "axis1": [ 1146, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 366, 33 ], "axis1": [ 1149, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 366, 33 ], "axis1": [ 1151, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 366, 33 ], "axis1": [ 1302, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 366, 33 ], "axis1": [ 1304, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 366, 33 ], "axis1": [ 1433, 24 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 366, 33 ], "axis1": [ 1458, 1 ], "matrix": "L15.mlp.W_out" } ], "l:516": [ { "axis0": [ 0, 1 ], "axis1": [ 892, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 892, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1535, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1034, 2 ], "axis1": [ 892, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 892, 2 ], "axis1": [ 1535, 1 ], "matrix": "L4.mlp.W_out" } ], "l:518": [ { "axis0": [ 0, 1 ], "axis1": [ 204, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 204, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 204, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1034, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 204, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 204, 2 ], "axis1": [ 1034, 1 ], "matrix": "L3.mlp.W_out" } ], "l:519": [ { "axis0": [ 0, 1 ], "axis1": [ 206, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 206, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 206, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1035, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 206, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 206, 2 ], "axis1": [ 1035, 1 ], "matrix": "L3.mlp.W_out" } ], "l:52": [ { "axis0": [ 0, 1 ], "axis1": [ 1835, 1 ], "matrix": "L1.mlp.W_out" } ], "l:520": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 1854, 64 ], "axis1": [ 128, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 67, 2 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 70, 13 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 84, 7 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 96, 3 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 130, 3 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 143, 3 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 1643, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 1857, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 1918, 35 ], "axis1": [ 192, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 67, 2 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 70, 13 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 84, 7 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 96, 3 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 130, 3 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 143, 3 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1643, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1857, 1 ], "matrix": "L15.attn.W_O" } ], "l:522": [ { "axis0": [ 0, 1 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 46, 1 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 70, 2 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 84, 8 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 336, 3 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 480, 16 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 882, 1 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 46, 1 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 68, 1 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 70, 2 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 84, 8 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 329, 1 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 336, 3 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 480, 16 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 882, 1 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 656, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 656, 33 ], "axis1": [ 1854, 33 ], "matrix": "L14.mlp.W_out" } ], "l:524": [ { "axis0": [ 0, 1 ], "axis1": [ 43, 14 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 43, 14 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 46, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 68, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 70, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 84, 8 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 329, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 336, 3 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 480, 16 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 882, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 868, 1 ], "axis1": [ 43, 14 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 43, 14 ], "axis1": [ 46, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 43, 14 ], "axis1": [ 68, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 43, 14 ], "axis1": [ 70, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 43, 14 ], "axis1": [ 84, 8 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 43, 14 ], "axis1": [ 329, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 43, 14 ], "axis1": [ 336, 3 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 43, 14 ], "axis1": [ 480, 16 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 43, 14 ], "axis1": [ 882, 1 ], "matrix": "L12.mlp.W_out" } ], "l:525": [ { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 964, 1 ], "axis1": [ 192, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 868, 1 ], "matrix": "L12.attn.W_O" } ], "l:526": [ { "axis0": [ 0, 1 ], "axis1": [ 249, 2 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 249, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1, 33 ], "axis1": [ 250, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 249, 1 ], "axis1": [ 868, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 250, 1 ], "axis1": [ 868, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" } ], "l:527": [ { "axis0": [ 0, 1 ], "axis1": [ 1881, 2 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 329, 1 ], "axis1": [ 1881, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 1882, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1881, 1 ], "axis1": [ 964, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1882, 1 ], "axis1": [ 964, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 964, 1 ], "matrix": "L11.mlp.W_out" } ], "l:528": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L10.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L10.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 329, 1 ], "matrix": "L10.mlp.W_out" }, { "axis0": [ 344, 2 ], "axis1": [ 1, 2 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 329, 1 ], "matrix": "L10.mlp.W_out" } ], "l:53": [ { "axis0": [ 0, 1 ], "axis1": [ 1836, 1 ], "matrix": "L1.mlp.W_out" } ], "l:530": [ { "axis0": [ 0, 1 ], "axis1": [ 107, 2 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 107, 2 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 345, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 868, 1 ], "axis1": [ 107, 2 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 107, 2 ], "axis1": [ 345, 1 ], "matrix": "L9.mlp.W_out" } ], "l:531": [ { "axis0": [ 0, 1 ], "axis1": [ 106, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 106, 1 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 344, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 172, 1 ], "axis1": [ 106, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 223, 12 ], "axis1": [ 106, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 237, 4 ], "axis1": [ 106, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 247, 16 ], "axis1": [ 106, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 344, 1 ], "matrix": "L9.mlp.W_out" } ], "l:532": [ { "axis0": [ 1028, 1 ], "axis1": [ 320, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 1028, 1 ], "axis1": [ 320, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 320, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 172, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 223, 12 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 237, 4 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 247, 16 ], "matrix": "L9.attn.W_O" } ], "l:533": [ { "axis0": [ 0, 1 ], "axis1": [ 1028, 1 ], "matrix": "L8.mlp.W_out" } ], "l:534": [ { "axis0": [ 0, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 55, 3 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 59, 2 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 130, 3 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 174, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 185, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 200, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 230, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 274, 3 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 278, 4 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 283, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 286, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 288, 2 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 339, 4 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 344, 2 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 669, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1025, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 55, 3 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 59, 2 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 67, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 130, 3 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 174, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 185, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 197, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 200, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 230, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 274, 3 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 278, 4 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 283, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 286, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 288, 2 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 339, 4 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 344, 2 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 669, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1025, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 689, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 689, 33 ], "axis1": [ 1887, 33 ], "matrix": "L14.mlp.W_out" } ], "l:536": [ { "axis0": [ 0, 1 ], "axis1": [ 341, 5 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 341, 5 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 55, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 59, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 67, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 130, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 174, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 185, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 197, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 200, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 230, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 274, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 278, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 283, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 286, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 288, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 339, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 344, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 669, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1025, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 346, 1 ], "axis1": [ 341, 5 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 341, 5 ], "axis1": [ 55, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 59, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 67, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 130, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 174, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 185, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 197, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 200, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 230, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 274, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 278, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 283, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 286, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 288, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 339, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 344, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 669, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 341, 5 ], "axis1": [ 1025, 1 ], "matrix": "L13.mlp.W_out" } ], "l:537": [ { "axis0": [ 0, 1 ], "axis1": [ 109, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 109, 1 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 346, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 263, 33 ], "axis1": [ 109, 1 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 109, 1 ], "axis1": [ 346, 1 ], "matrix": "L9.mlp.W_out" } ], "l:538": [ { "axis0": [ 1029, 1 ], "axis1": [ 384, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 1029, 1 ], "axis1": [ 384, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 384, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 263, 33 ], "matrix": "L9.attn.W_O" } ], "l:539": [ { "axis0": [ 0, 1 ], "axis1": [ 1029, 1 ], "matrix": "L8.mlp.W_out" } ], "l:540": [ { "axis0": [ 0, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 49, 3 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 53, 2 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 223, 5 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 231, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 245, 2 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 254, 13 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 268, 3 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 273, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 868, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1150, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 49, 3 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 53, 2 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 173, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 223, 5 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 231, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 245, 2 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 254, 13 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 268, 3 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 273, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 868, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1150, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 722, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 722, 33 ], "axis1": [ 1920, 33 ], "matrix": "L14.mlp.W_out" } ], "l:542": [ { "axis0": [ 0, 1 ], "axis1": [ 135, 206 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 135, 206 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 49, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 53, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 173, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 223, 5 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 231, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 245, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 254, 13 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 268, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 273, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 868, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1150, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1829, 1 ], "axis1": [ 135, 206 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 135, 206 ], "axis1": [ 49, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 135, 206 ], "axis1": [ 53, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 135, 206 ], "axis1": [ 173, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 135, 206 ], "axis1": [ 223, 5 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 135, 206 ], "axis1": [ 231, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 135, 206 ], "axis1": [ 245, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 135, 206 ], "axis1": [ 254, 13 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 135, 206 ], "axis1": [ 268, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 135, 206 ], "axis1": [ 273, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 135, 206 ], "axis1": [ 868, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 135, 206 ], "axis1": [ 1150, 1 ], "matrix": "L13.mlp.W_out" } ], "l:543": [ { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 1085, 1 ], "axis1": [ 320, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 1829, 1 ], "matrix": "L13.attn.W_O" } ], "l:544": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1724, 33 ], "axis1": [ 64, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1724, 33 ], "axis1": [ 65, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 64, 1 ], "axis1": [ 1829, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 65, 1 ], "axis1": [ 1829, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:545": [ { "axis0": [ 158, 1 ], "axis1": [ 1856, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 158, 1 ], "axis1": [ 1856, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1856, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1856, 64 ], "axis1": [ 1724, 33 ], "matrix": "L1.attn.W_O" } ], "l:546": [ { "axis0": [ 0, 1 ], "axis1": [ 158, 1 ], "matrix": "L0.mlp.W_out" } ], "l:547": [ { "axis0": [ 0, 1 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1613, 25 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1639, 4 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1644, 1 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1827, 1 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1833, 2 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1841, 1 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1613, 25 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1639, 4 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1644, 1 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1827, 1 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1833, 2 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1841, 1 ], "axis1": [ 232, 33 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 232, 33 ], "axis1": [ 506, 33 ], "matrix": "L16.mlp.W_out" } ], "l:549": [ { "axis0": [ 0, 1 ], "axis1": [ 894, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 894, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1841, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1036, 2 ], "axis1": [ 894, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 894, 2 ], "axis1": [ 1841, 1 ], "matrix": "L4.mlp.W_out" } ], "l:55": [ { "axis0": [ 0, 1 ], "axis1": [ 1837, 1 ], "matrix": "L1.mlp.W_out" } ], "l:551": [ { "axis0": [ 0, 1 ], "axis1": [ 208, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 208, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 208, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1036, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 208, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 208, 2 ], "axis1": [ 1036, 1 ], "matrix": "L3.mlp.W_out" } ], "l:552": [ { "axis0": [ 0, 1 ], "axis1": [ 210, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 210, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 210, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1037, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 210, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 210, 2 ], "axis1": [ 1037, 1 ], "matrix": "L3.mlp.W_out" } ], "l:553": [ { "axis0": [ 0, 1 ], "axis1": [ 498, 66 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 353, 66 ], "axis1": [ 498, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1953, 33 ], "axis1": [ 498, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 353, 66 ], "axis1": [ 531, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1953, 33 ], "axis1": [ 531, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 498, 25 ], "axis1": [ 1613, 25 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 523, 4 ], "axis1": [ 1639, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 527, 1 ], "axis1": [ 1644, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 528, 1 ], "axis1": [ 1827, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 529, 2 ], "axis1": [ 1833, 2 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 531, 25 ], "axis1": [ 1613, 25 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 556, 4 ], "axis1": [ 1639, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 560, 1 ], "axis1": [ 1644, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 561, 1 ], "axis1": [ 1827, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 562, 2 ], "axis1": [ 1833, 2 ], "diagonal": true, "matrix": "L15.mlp.W_out" } ], "l:555": [ { "axis0": [ 0, 1 ], "axis1": [ 151, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 151, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 47, 1 ], "axis1": [ 151, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1184, 31 ], "axis1": [ 151, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1675, 1 ], "axis1": [ 151, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 151, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 47, 1 ], "axis1": [ 151, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1184, 31 ], "axis1": [ 151, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1675, 1 ], "axis1": [ 151, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 151, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 151, 33 ], "axis1": [ 353, 33 ], "matrix": "L5.mlp.W_out" } ], "l:557": [ { "axis0": [ 0, 1 ], "axis1": [ 200, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 200, 14 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 47, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1184, 31 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1675, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 49, 1 ], "axis1": [ 200, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 200, 14 ], "axis1": [ 47, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 200, 14 ], "axis1": [ 1184, 31 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 200, 14 ], "axis1": [ 1675, 1 ], "matrix": "L4.mlp.W_out" } ], "l:558": [ { "axis0": [ 0, 1 ], "axis1": [ 294, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 837, 1 ], "axis1": [ 294, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 837, 1 ], "axis1": [ 295, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 294, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 295, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 294, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 295, 1 ], "matrix": "L3.mlp.W_in" } ], "l:559": [ { "axis0": [ 0, 1 ], "axis1": [ 25, 2 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 25, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 1, 33 ], "axis1": [ 26, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 25, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" }, { "axis0": [ 26, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" } ], "l:56": [ { "axis0": [ 0, 1 ], "axis1": [ 162, 1 ], "matrix": "L0.mlp.W_out" } ], "l:560": [ { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 999, 1 ], "axis1": [ 512, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 837, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 837, 1 ], "matrix": "L3.mlp.W_out" } ], "l:561": [ { "axis0": [ 0, 1 ], "axis1": [ 121, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 121, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 999, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1791, 1 ], "axis1": [ 121, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1807, 1 ], "axis1": [ 121, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 121, 2 ], "axis1": [ 999, 1 ], "matrix": "L2.mlp.W_out" } ], "l:563": [ { "axis0": [ 0, 1 ], "axis1": [ 20, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 20, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1807, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 49, 1 ], "axis1": [ 20, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 20, 2 ], "axis1": [ 1807, 1 ], "matrix": "L1.mlp.W_out" } ], "l:564": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 1 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1791, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 1196, 33 ], "axis1": [ 1, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1, 1 ], "axis1": [ 1791, 1 ], "matrix": "L1.mlp.W_out" } ], "l:565": [ { "axis0": [ 133, 1 ], "axis1": [ 832, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 133, 1 ], "axis1": [ 832, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 832, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 1196, 33 ], "matrix": "L1.attn.W_O" } ], "l:566": [ { "axis0": [ 0, 1 ], "axis1": [ 133, 1 ], "matrix": "L0.mlp.W_out" } ], "l:567": [ { "axis0": [ 0, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 46, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 73, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1003, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1009, 5 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1015, 3 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1068, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1079, 2 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1082, 3 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1636, 3 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1644, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1668, 7 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1695, 2 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1702, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1708, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1843, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 46, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 73, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1003, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1009, 5 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1015, 3 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1068, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1079, 2 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1082, 3 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1636, 3 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1644, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1668, 7 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1695, 2 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1702, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1708, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1843, 1 ], "axis1": [ 184, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 184, 33 ], "axis1": [ 386, 33 ], "matrix": "L5.mlp.W_out" } ], "l:569": [ { "axis0": [ 0, 1 ], "axis1": [ 186, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 186, 14 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 40, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 46, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 73, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1003, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1009, 5 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1015, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1068, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1079, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1082, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1636, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1644, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1668, 7 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1695, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1702, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1708, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1843, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 35, 1 ], "axis1": [ 186, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 186, 14 ], "axis1": [ 40, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 46, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 73, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 1003, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 1009, 5 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 1015, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 1068, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 1079, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 1082, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 1636, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 1644, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 1668, 7 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 1695, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 1702, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 1708, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 186, 14 ], "axis1": [ 1843, 1 ], "matrix": "L4.mlp.W_out" } ], "l:57": [ { "axis0": [ 0, 1 ], "axis1": [ 44, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1085, 26 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1112, 6 ], "matrix": "L3.mlp.W_out" } ], "l:570": [ { "axis0": [ 0, 1 ], "axis1": [ 298, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 839, 1 ], "axis1": [ 298, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 839, 1 ], "axis1": [ 299, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 298, 1 ], "axis1": [ 35, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 299, 1 ], "axis1": [ 35, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 298, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 299, 1 ], "matrix": "L3.mlp.W_in" } ], "l:571": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L0.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L0.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 64, 64 ], "matrix": "L0.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 35, 1 ], "matrix": "L0.attn.W_O" } ], "l:572": [ { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1001, 1 ], "axis1": [ 640, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 839, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 839, 1 ], "matrix": "L3.mlp.W_out" } ], "l:573": [ { "axis0": [ 0, 1 ], "axis1": [ 125, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 125, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1001, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 46, 1 ], "axis1": [ 125, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1793, 1 ], "axis1": [ 125, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 125, 2 ], "axis1": [ 1001, 1 ], "matrix": "L2.mlp.W_out" } ], "l:575": [ { "axis0": [ 0, 1 ], "axis1": [ 19, 2 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 19, 2 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 46, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 35, 1 ], "axis1": [ 19, 2 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 19, 2 ], "axis1": [ 46, 1 ], "matrix": "L0.mlp.W_out" } ], "l:576": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 1 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1793, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 1262, 33 ], "axis1": [ 3, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 3, 1 ], "axis1": [ 1793, 1 ], "matrix": "L1.mlp.W_out" } ], "l:577": [ { "axis0": [ 135, 1 ], "axis1": [ 960, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 135, 1 ], "axis1": [ 960, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 960, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 960, 64 ], "axis1": [ 1262, 33 ], "matrix": "L1.attn.W_O" } ], "l:578": [ { "axis0": [ 0, 1 ], "axis1": [ 135, 1 ], "matrix": "L0.mlp.W_out" } ], "l:579": [ { "axis0": [ 0, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 34, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 36, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 44, 2 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 83, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 143, 9 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 158, 7 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 168, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 171, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 178, 7 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 229, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 332, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 34, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 36, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 44, 2 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 83, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 143, 9 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 158, 7 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 168, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 171, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 178, 7 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 222, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 229, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 332, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 755, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 755, 33 ], "axis1": [ 1953, 33 ], "matrix": "L14.mlp.W_out" } ], "l:58": [ { "axis0": [ 0, 1 ], "axis1": [ 247, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 247, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 234, 33 ], "axis1": [ 247, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 247, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 234, 33 ], "axis1": [ 247, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 247, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 247, 33 ], "axis1": [ 75, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 247, 33 ], "axis1": [ 1248, 16 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 247, 33 ], "axis1": [ 1265, 16 ], "matrix": "L4.mlp.W_out" } ], "l:581": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 134 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 134 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 34, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 36, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 44, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 83, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 143, 9 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 158, 7 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 168, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 171, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 178, 7 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 222, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 229, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 332, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1830, 1 ], "axis1": [ 1, 134 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1, 134 ], "axis1": [ 34, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 134 ], "axis1": [ 36, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 134 ], "axis1": [ 44, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 134 ], "axis1": [ 83, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 134 ], "axis1": [ 143, 9 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 134 ], "axis1": [ 158, 7 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 134 ], "axis1": [ 168, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 134 ], "axis1": [ 171, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 134 ], "axis1": [ 178, 7 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 134 ], "axis1": [ 222, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 134 ], "axis1": [ 229, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 134 ], "axis1": [ 332, 1 ], "matrix": "L13.mlp.W_out" } ], "l:582": [ { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 1084, 1 ], "axis1": [ 384, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 1830, 1 ], "matrix": "L13.attn.W_O" } ], "l:583": [ { "axis0": [ 0, 1 ], "axis1": [ 66, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1757, 33 ], "axis1": [ 66, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1757, 33 ], "axis1": [ 67, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 1830, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 67, 1 ], "axis1": [ 1830, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:584": [ { "axis0": [ 159, 1 ], "axis1": [ 1920, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 159, 1 ], "axis1": [ 1920, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1920, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1920, 64 ], "axis1": [ 1757, 33 ], "matrix": "L1.attn.W_O" } ], "l:585": [ { "axis0": [ 0, 1 ], "axis1": [ 159, 1 ], "matrix": "L0.mlp.W_out" } ], "l:586": [ { "axis0": [ 0, 1 ], "axis1": [ 399, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 399, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 106, 24 ], "axis1": [ 399, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 133, 7 ], "axis1": [ 399, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 186, 2 ], "axis1": [ 399, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1457, 1 ], "axis1": [ 399, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 106, 24 ], "axis1": [ 399, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 133, 7 ], "axis1": [ 399, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 186, 2 ], "axis1": [ 399, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1457, 1 ], "axis1": [ 399, 33 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 399, 33 ], "axis1": [ 1459, 28 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 399, 33 ], "axis1": [ 1569, 5 ], "matrix": "L15.mlp.W_out" } ], "l:588": [ { "axis0": [ 0, 1 ], "axis1": [ 896, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1457, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1038, 2 ], "axis1": [ 896, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 896, 2 ], "axis1": [ 1457, 1 ], "matrix": "L4.mlp.W_out" } ], "l:590": [ { "axis0": [ 0, 1 ], "axis1": [ 212, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 212, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 212, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1038, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 212, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 212, 2 ], "axis1": [ 1038, 1 ], "matrix": "L3.mlp.W_out" } ], "l:591": [ { "axis0": [ 0, 1 ], "axis1": [ 214, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 214, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 214, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1039, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 214, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 214, 2 ], "axis1": [ 1039, 1 ], "matrix": "L3.mlp.W_out" } ], "l:592": [ { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 94, 15 ], "axis1": [ 512, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 115, 3 ], "axis1": [ 512, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 120, 10 ], "axis1": [ 512, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 241, 1 ], "axis1": [ 512, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 330, 1 ], "axis1": [ 512, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 477, 1 ], "axis1": [ 512, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 516, 33 ], "axis1": [ 512, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 106, 24 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 133, 7 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 186, 2 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 478, 2 ], "axis1": [ 576, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1554, 4 ], "axis1": [ 576, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1559, 29 ], "axis1": [ 576, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 106, 24 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 133, 7 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 186, 2 ], "matrix": "L14.attn.W_O" } ], "l:594": [ { "axis0": [ 0, 1 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 44, 1 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 46, 4 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 51, 4 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 61, 2 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 66, 2 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 71, 10 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 83, 1 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 85, 7 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 44, 1 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 46, 4 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 51, 4 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 58, 1 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 61, 2 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 66, 2 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 69, 1 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 71, 10 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 83, 1 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 85, 7 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 18, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 18, 33 ], "axis1": [ 516, 33 ], "matrix": "L8.mlp.W_out" } ], "l:596": [ { "axis0": [ 0, 1 ], "axis1": [ 26, 14 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 26, 14 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 44, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 46, 4 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 51, 4 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 58, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 61, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 66, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 71, 10 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 83, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 85, 7 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 50, 1 ], "axis1": [ 26, 14 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 26, 14 ], "axis1": [ 44, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 26, 14 ], "axis1": [ 46, 4 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 26, 14 ], "axis1": [ 51, 4 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 26, 14 ], "axis1": [ 58, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 26, 14 ], "axis1": [ 61, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 26, 14 ], "axis1": [ 66, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 26, 14 ], "axis1": [ 69, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 26, 14 ], "axis1": [ 71, 10 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 26, 14 ], "axis1": [ 83, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 26, 14 ], "axis1": [ 85, 7 ], "matrix": "L7.mlp.W_out" } ], "l:597": [ { "axis0": [ 0, 1 ], "axis1": [ 296, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 838, 1 ], "axis1": [ 296, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 838, 1 ], "axis1": [ 297, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 296, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 297, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 296, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 297, 1 ], "matrix": "L3.mlp.W_in" } ], "l:598": [ { "axis0": [ 0, 1 ], "axis1": [ 27, 2 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 27, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 1, 33 ], "axis1": [ 28, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" }, { "axis0": [ 28, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" } ], "l:599": [ { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1000, 1 ], "axis1": [ 576, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 838, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 838, 1 ], "matrix": "L3.mlp.W_out" } ], "l:60": [ { "axis0": [ 163, 1 ], "axis1": [ 384, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 1083, 1 ], "axis1": [ 384, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 164, 1 ], "axis1": [ 384, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1796, 1 ], "axis1": [ 384, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 39, 1 ], "axis1": [ 384, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 1638, 1 ], "matrix": "L4.attn.W_O" } ], "l:600": [ { "axis0": [ 0, 1 ], "axis1": [ 123, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 123, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1000, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1792, 1 ], "axis1": [ 123, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1808, 1 ], "axis1": [ 123, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 123, 2 ], "axis1": [ 1000, 1 ], "matrix": "L2.mlp.W_out" } ], "l:602": [ { "axis0": [ 0, 1 ], "axis1": [ 22, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 22, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1808, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 50, 1 ], "axis1": [ 22, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 22, 2 ], "axis1": [ 1808, 1 ], "matrix": "L1.mlp.W_out" } ], "l:603": [ { "axis0": [ 0, 1 ], "axis1": [ 2, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 2, 1 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1792, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 1229, 33 ], "axis1": [ 2, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 2, 1 ], "axis1": [ 1792, 1 ], "matrix": "L1.mlp.W_out" } ], "l:604": [ { "axis0": [ 134, 1 ], "axis1": [ 896, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 134, 1 ], "axis1": [ 896, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 896, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 1229, 33 ], "matrix": "L1.attn.W_O" } ], "l:605": [ { "axis0": [ 0, 1 ], "axis1": [ 134, 1 ], "matrix": "L0.mlp.W_out" } ], "l:606": [ { "axis0": [ 0, 1 ], "axis1": [ 143, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 143, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 483, 33 ], "axis1": [ 143, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 143, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 483, 33 ], "axis1": [ 143, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 143, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 143, 33 ], "axis1": [ 94, 15 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 143, 33 ], "axis1": [ 115, 3 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 143, 33 ], "axis1": [ 120, 10 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 143, 33 ], "axis1": [ 241, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 143, 33 ], "axis1": [ 330, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 143, 33 ], "axis1": [ 477, 3 ], "matrix": "L9.mlp.W_out" } ], "l:608": [ { "axis0": [ 0, 1 ], "axis1": [ 4, 14 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 4, 14 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 483, 33 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 36, 1 ], "axis1": [ 4, 14 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 4, 14 ], "axis1": [ 483, 33 ], "matrix": "L8.mlp.W_out" } ], "l:609": [ { "axis0": [ 0, 1 ], "axis1": [ 300, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 1795, 1 ], "axis1": [ 300, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1795, 1 ], "axis1": [ 301, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 300, 1 ], "axis1": [ 36, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 301, 1 ], "axis1": [ 36, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 300, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 301, 1 ], "matrix": "L3.mlp.W_in" } ], "l:610": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L0.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L0.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 128, 64 ], "matrix": "L0.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 36, 1 ], "matrix": "L0.attn.W_O" } ], "l:611": [ { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1002, 1 ], "axis1": [ 704, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 1795, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1795, 1 ], "matrix": "L3.mlp.W_out" } ], "l:612": [ { "axis0": [ 0, 1 ], "axis1": [ 127, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 127, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1002, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 47, 1 ], "axis1": [ 127, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1794, 1 ], "axis1": [ 127, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 127, 2 ], "axis1": [ 1002, 1 ], "matrix": "L2.mlp.W_out" } ], "l:614": [ { "axis0": [ 0, 1 ], "axis1": [ 21, 2 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 21, 2 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 47, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 36, 1 ], "axis1": [ 21, 2 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 21, 2 ], "axis1": [ 47, 1 ], "matrix": "L0.mlp.W_out" } ], "l:615": [ { "axis0": [ 0, 1 ], "axis1": [ 4, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 4, 1 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1794, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 1295, 33 ], "axis1": [ 4, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 4, 1 ], "axis1": [ 1794, 1 ], "matrix": "L1.mlp.W_out" } ], "l:616": [ { "axis0": [ 136, 1 ], "axis1": [ 1024, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 136, 1 ], "axis1": [ 1024, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1024, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1024, 64 ], "axis1": [ 1295, 33 ], "matrix": "L1.attn.W_O" } ], "l:617": [ { "axis0": [ 0, 1 ], "axis1": [ 136, 1 ], "matrix": "L0.mlp.W_out" } ], "l:618": [ { "axis0": [ 0, 1 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 61, 5 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 297, 10 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 313, 4 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 318, 3 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 322, 1 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 324, 3 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 333, 2 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 61, 5 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 297, 10 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 308, 4 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 313, 4 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 318, 3 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 322, 1 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 324, 3 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 328, 1 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 333, 2 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 942, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 942, 33 ], "axis1": [ 1554, 4 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 942, 33 ], "axis1": [ 1559, 29 ], "matrix": "L13.mlp.W_out" } ], "l:62": [ { "axis0": [ 0, 1 ], "axis1": [ 1083, 1 ], "matrix": "L3.mlp.W_out" } ], "l:620": [ { "axis0": [ 0, 1 ], "axis1": [ 25, 18 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 25, 18 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 61, 5 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 297, 10 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 308, 4 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 313, 4 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 318, 3 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 322, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 324, 3 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 328, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 333, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 52, 1 ], "axis1": [ 25, 18 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 25, 18 ], "axis1": [ 61, 5 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 25, 18 ], "axis1": [ 297, 10 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 25, 18 ], "axis1": [ 308, 4 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 25, 18 ], "axis1": [ 313, 4 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 25, 18 ], "axis1": [ 318, 3 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 25, 18 ], "axis1": [ 322, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 25, 18 ], "axis1": [ 324, 3 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 25, 18 ], "axis1": [ 328, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 25, 18 ], "axis1": [ 333, 2 ], "matrix": "L12.mlp.W_out" } ], "l:621": [ { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 673, 2 ], "axis1": [ 256, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 52, 1 ], "matrix": "L12.attn.W_O" } ], "l:622": [ { "axis0": [ 0, 1 ], "axis1": [ 450, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 450, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 180, 21 ], "axis1": [ 450, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 234, 12 ], "axis1": [ 450, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1586, 1 ], "axis1": [ 450, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 180, 21 ], "axis1": [ 450, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 234, 12 ], "axis1": [ 450, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1586, 1 ], "axis1": [ 450, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 450, 33 ], "axis1": [ 750, 12 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 450, 33 ], "axis1": [ 795, 21 ], "matrix": "L5.mlp.W_out" } ], "l:624": [ { "axis0": [ 0, 1 ], "axis1": [ 898, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 898, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1586, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1040, 1 ], "axis1": [ 898, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1835, 1 ], "axis1": [ 898, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 898, 2 ], "axis1": [ 1586, 1 ], "matrix": "L4.mlp.W_out" } ], "l:626": [ { "axis0": [ 0, 1 ], "axis1": [ 216, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 216, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 216, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1040, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 216, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 216, 2 ], "axis1": [ 1040, 1 ], "matrix": "L3.mlp.W_out" } ], "l:627": [ { "axis0": [ 0, 1 ], "axis1": [ 218, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 218, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 218, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1835, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 218, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 218, 2 ], "axis1": [ 1835, 1 ], "matrix": "L3.mlp.W_out" } ], "l:628": [ { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 76, 1 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1380, 2 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1383, 3 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1387, 2 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1390, 17 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1408, 24 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1433, 12 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1639, 1 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1683, 2 ], "axis1": [ 256, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 180, 21 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 234, 12 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 58, 1 ], "axis1": [ 320, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1445, 12 ], "axis1": [ 320, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1458, 3 ], "axis1": [ 320, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1462, 17 ], "axis1": [ 320, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1685, 2 ], "axis1": [ 320, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 180, 21 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 234, 12 ], "matrix": "L5.attn.W_O" } ], "l:63": [ { "axis0": [ 0, 1 ], "axis1": [ 163, 1 ], "matrix": "L0.mlp.W_out" } ], "l:630": [ { "axis0": [ 0, 1 ], "axis1": [ 379, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 379, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 168, 33 ], "axis1": [ 379, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 379, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 168, 33 ], "axis1": [ 379, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 379, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 379, 33 ], "axis1": [ 76, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 379, 33 ], "axis1": [ 1380, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 379, 33 ], "axis1": [ 1383, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 379, 33 ], "axis1": [ 1387, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 379, 33 ], "axis1": [ 1390, 17 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 379, 33 ], "axis1": [ 1408, 5 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 379, 33 ], "axis1": [ 1639, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 379, 33 ], "axis1": [ 1683, 2 ], "matrix": "L4.mlp.W_out" } ], "l:632": [ { "axis0": [ 0, 1 ], "axis1": [ 168, 33 ], "matrix": "L0.mlp.W_out" } ], "l:633": [ { "axis0": [ 0, 1 ], "axis1": [ 412, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 412, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 267, 33 ], "axis1": [ 412, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 412, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 267, 33 ], "axis1": [ 412, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 412, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 412, 33 ], "axis1": [ 1413, 19 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 412, 33 ], "axis1": [ 1433, 13 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 412, 33 ], "axis1": [ 1685, 1 ], "matrix": "L4.mlp.W_out" } ], "l:635": [ { "axis0": [ 0, 1 ], "axis1": [ 267, 33 ], "matrix": "L0.mlp.W_out" } ], "l:636": [ { "axis0": [ 0, 1 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 72, 1 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 888, 1 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 891, 3 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 896, 14 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 911, 9 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1805, 3 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1809, 1 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1811, 1 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 72, 1 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 888, 1 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 891, 3 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 896, 14 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 911, 9 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1805, 3 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1809, 1 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1811, 1 ], "axis1": [ 445, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 445, 33 ], "axis1": [ 58, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 445, 33 ], "axis1": [ 1446, 11 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 445, 33 ], "axis1": [ 1458, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 445, 33 ], "axis1": [ 1462, 17 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 445, 33 ], "axis1": [ 1686, 1 ], "matrix": "L4.mlp.W_out" } ], "l:638": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 5 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 5 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 72, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 888, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 891, 3 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 14 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 911, 9 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1805, 3 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1809, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1811, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 1010, 1 ], "axis1": [ 128, 5 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 128, 5 ], "axis1": [ 72, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 128, 5 ], "axis1": [ 888, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 128, 5 ], "axis1": [ 891, 3 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 128, 5 ], "axis1": [ 896, 14 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 128, 5 ], "axis1": [ 911, 9 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 128, 5 ], "axis1": [ 1805, 3 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 128, 5 ], "axis1": [ 1809, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 128, 5 ], "axis1": [ 1811, 1 ], "matrix": "L3.mlp.W_out" } ], "l:639": [ { "axis0": [ 0, 1 ], "axis1": [ 142, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 142, 1 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1010, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 947, 33 ], "axis1": [ 142, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1010, 1 ], "matrix": "L2.mlp.W_out" } ], "l:640": [ { "axis0": [ 1844, 1 ], "axis1": [ 768, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 1844, 1 ], "axis1": [ 768, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 768, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 947, 33 ], "matrix": "L2.attn.W_O" } ], "l:641": [ { "axis0": [ 0, 1 ], "axis1": [ 1844, 1 ], "matrix": "L1.mlp.W_out" } ], "l:642": [ { "axis0": [ 0, 1 ], "axis1": [ 51, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 51, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 180, 33 ], "axis1": [ 51, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1844, 1 ], "axis1": [ 51, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 180, 33 ], "axis1": [ 51, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1844, 1 ], "axis1": [ 51, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 51, 33 ], "axis1": [ 549, 33 ], "matrix": "L8.mlp.W_out" } ], "l:644": [ { "axis0": [ 0, 1 ], "axis1": [ 900, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 900, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1844, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1042, 2 ], "axis1": [ 900, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 900, 2 ], "axis1": [ 1844, 1 ], "matrix": "L4.mlp.W_out" } ], "l:646": [ { "axis0": [ 0, 1 ], "axis1": [ 220, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 220, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 220, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1042, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 220, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 220, 2 ], "axis1": [ 1042, 1 ], "matrix": "L3.mlp.W_out" } ], "l:647": [ { "axis0": [ 0, 1 ], "axis1": [ 222, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 222, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 222, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1043, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 222, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 222, 2 ], "axis1": [ 1043, 1 ], "matrix": "L3.mlp.W_out" } ], "l:648": [ { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 92, 26 ], "axis1": [ 192, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 120, 1 ], "axis1": [ 192, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 135, 37 ], "axis1": [ 192, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 180, 33 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 172, 2 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 419, 13 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 465, 20 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 180, 33 ], "matrix": "L8.attn.W_O" } ], "l:65": [ { "axis0": [ 0, 1 ], "axis1": [ 164, 1 ], "matrix": "L0.mlp.W_out" } ], "l:650": [ { "axis0": [ 0, 1 ], "axis1": [ 40, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 40, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1223, 33 ], "axis1": [ 40, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 40, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1223, 33 ], "axis1": [ 40, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 40, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 40, 33 ], "axis1": [ 92, 26 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 40, 33 ], "axis1": [ 120, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 40, 33 ], "axis1": [ 135, 6 ], "matrix": "L7.mlp.W_out" } ], "l:652": [ { "axis0": [ 0, 1 ], "axis1": [ 70, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 70, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 299, 24 ], "axis1": [ 70, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 324, 9 ], "axis1": [ 70, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 935, 33 ], "axis1": [ 70, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1113, 1 ], "axis1": [ 70, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 299, 24 ], "axis1": [ 70, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 324, 9 ], "axis1": [ 70, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 935, 33 ], "axis1": [ 70, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1113, 1 ], "axis1": [ 70, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 70, 66 ], "axis1": [ 1223, 33 ], "matrix": "L6.mlp.W_out" } ], "l:654": [ { "axis0": [ 0, 1 ], "axis1": [ 1113, 1 ], "matrix": "L5.mlp.W_out" } ], "l:655": [ { "axis0": [ 0, 1 ], "axis1": [ 935, 33 ], "matrix": "L5.mlp.W_out" } ], "l:656": [ { "axis0": [ 347, 4 ], "axis1": [ 384, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 312, 4 ], "axis1": [ 384, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 384, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 299, 24 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 324, 9 ], "matrix": "L6.attn.W_O" } ], "l:657": [ { "axis0": [ 0, 1 ], "axis1": [ 127, 14 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 127, 14 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 347, 4 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 1073, 1 ], "axis1": [ 127, 14 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 127, 14 ], "axis1": [ 347, 4 ], "matrix": "L5.mlp.W_out" } ], "l:658": [ { "axis0": [ 0, 1 ], "axis1": [ 282, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 46, 3 ], "axis1": [ 282, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 83, 1 ], "axis1": [ 282, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1644, 5 ], "axis1": [ 282, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1694, 1 ], "axis1": [ 282, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1697, 1 ], "axis1": [ 282, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1708, 22 ], "axis1": [ 282, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 46, 3 ], "axis1": [ 283, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 83, 1 ], "axis1": [ 283, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1644, 5 ], "axis1": [ 283, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1694, 1 ], "axis1": [ 283, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1697, 1 ], "axis1": [ 283, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1708, 22 ], "axis1": [ 283, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 282, 1 ], "axis1": [ 1073, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 283, 1 ], "axis1": [ 1073, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" } ], "l:659": [ { "axis0": [ 145, 1 ], "axis1": [ 128, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 145, 1 ], "axis1": [ 128, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 128, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 46, 3 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 83, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 1644, 5 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 1694, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 1697, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 1708, 22 ], "matrix": "L3.attn.W_O" } ], "l:66": [ { "axis0": [ 0, 1 ], "axis1": [ 234, 33 ], "matrix": "L0.mlp.W_out" } ], "l:660": [ { "axis0": [ 0, 1 ], "axis1": [ 145, 1 ], "matrix": "L0.mlp.W_out" } ], "l:661": [ { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 1645, 1 ], "axis1": [ 640, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1712, 3 ], "axis1": [ 640, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 312, 4 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 312, 4 ], "matrix": "L5.mlp.W_out" } ], "l:662": [ { "axis0": [ 0, 1 ], "axis1": [ 38, 16 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 38, 16 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 38, 16 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1645, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1712, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 67, 1 ], "axis1": [ 38, 16 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1069, 1 ], "axis1": [ 38, 16 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 38, 16 ], "axis1": [ 1645, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 38, 16 ], "axis1": [ 1712, 3 ], "matrix": "L4.mlp.W_out" } ], "l:664": [ { "axis0": [ 0, 1 ], "axis1": [ 274, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 836, 1 ], "axis1": [ 274, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 836, 1 ], "axis1": [ 275, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 274, 1 ], "axis1": [ 1069, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 275, 1 ], "axis1": [ 1069, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 274, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 275, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1069, 1 ], "matrix": "L3.mlp.W_out" } ], "l:665": [ { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1069, 1 ], "axis1": [ 576, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 67, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 67, 1 ], "matrix": "L4.mlp.W_out" } ], "l:666": [ { "axis0": [ 0, 1 ], "axis1": [ 73, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 73, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1423, 9 ], "axis1": [ 73, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1433, 24 ], "axis1": [ 73, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 73, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1423, 9 ], "axis1": [ 73, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1433, 24 ], "axis1": [ 73, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 73, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 73, 33 ], "axis1": [ 141, 33 ], "matrix": "L7.mlp.W_out" } ], "l:668": [ { "axis0": [ 0, 1 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 698, 18 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 828, 6 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 836, 4 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 841, 4 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 847, 1 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1008, 1 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1028, 13 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1042, 4 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1047, 3 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1052, 9 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1062, 1 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1064, 3 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 698, 18 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 828, 6 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 836, 4 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 841, 4 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 847, 1 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1008, 1 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1028, 13 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1042, 4 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1047, 3 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1052, 9 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1062, 1 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1064, 3 ], "axis1": [ 334, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 334, 66 ], "axis1": [ 1423, 9 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 334, 66 ], "axis1": [ 1433, 24 ], "matrix": "L6.mlp.W_out" } ], "l:67": [ { "axis0": [ 0, 1 ], "axis1": [ 280, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 280, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 300, 33 ], "axis1": [ 280, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 280, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 300, 33 ], "axis1": [ 280, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 280, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 280, 33 ], "axis1": [ 1281, 33 ], "matrix": "L4.mlp.W_out" } ], "l:670": [ { "axis0": [ 0, 1 ], "axis1": [ 139, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 139, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1008, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1819, 1 ], "axis1": [ 139, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 139, 2 ], "axis1": [ 1008, 1 ], "matrix": "L2.mlp.W_out" } ], "l:671": [ { "axis0": [ 0, 1 ], "axis1": [ 44, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1097, 33 ], "axis1": [ 44, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1097, 33 ], "axis1": [ 45, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 44, 1 ], "axis1": [ 1819, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 45, 1 ], "axis1": [ 1819, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:672": [ { "axis0": [ 129, 1 ], "axis1": [ 640, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 129, 1 ], "axis1": [ 640, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 640, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 1097, 33 ], "matrix": "L1.attn.W_O" } ], "l:673": [ { "axis0": [ 0, 1 ], "axis1": [ 129, 1 ], "matrix": "L0.mlp.W_out" } ], "l:674": [ { "axis0": [ 344, 3 ], "axis1": [ 640, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 316, 3 ], "axis1": [ 640, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 640, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 698, 18 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 828, 6 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 836, 4 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 841, 4 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 847, 1 ], "matrix": "L6.attn.W_O" } ], "l:675": [ { "axis0": [ 0, 1 ], "axis1": [ 109, 18 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 109, 18 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 83, 1 ], "axis1": [ 109, 18 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 109, 18 ], "axis1": [ 344, 3 ], "matrix": "L5.mlp.W_out" } ], "l:676": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 998, 1 ], "axis1": [ 0, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 83, 1 ], "matrix": "L5.attn.W_O" } ], "l:677": [ { "axis0": [ 0, 1 ], "axis1": [ 35, 3 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 3 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 137, 1 ], "axis1": [ 35, 3 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1071, 1 ], "axis1": [ 35, 3 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 35, 3 ], "axis1": [ 83, 1 ], "matrix": "L4.mlp.W_out" } ], "l:679": [ { "axis0": [ 0, 1 ], "axis1": [ 278, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 1636, 3 ], "axis1": [ 278, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1640, 1 ], "axis1": [ 278, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1643, 1 ], "axis1": [ 278, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1664, 17 ], "axis1": [ 278, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1688, 6 ], "axis1": [ 278, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1695, 2 ], "axis1": [ 278, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1702, 2 ], "axis1": [ 278, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1706, 1 ], "axis1": [ 278, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1636, 3 ], "axis1": [ 279, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1640, 1 ], "axis1": [ 279, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1643, 1 ], "axis1": [ 279, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1664, 17 ], "axis1": [ 279, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1688, 6 ], "axis1": [ 279, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1695, 2 ], "axis1": [ 279, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1702, 2 ], "axis1": [ 279, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1706, 1 ], "axis1": [ 279, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 278, 1 ], "axis1": [ 1071, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 279, 1 ], "axis1": [ 1071, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" } ], "l:680": [ { "axis0": [ 124, 1 ], "axis1": [ 192, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 124, 1 ], "axis1": [ 192, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 192, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 1636, 3 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1640, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1643, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1664, 17 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1688, 6 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1695, 2 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1702, 2 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1706, 1 ], "matrix": "L2.attn.W_O" } ], "l:681": [ { "axis0": [ 0, 1 ], "axis1": [ 124, 1 ], "matrix": "L0.mlp.W_out" } ], "l:682": [ { "axis0": [ 0, 1 ], "axis1": [ 137, 1 ], "matrix": "L0.mlp.W_out" } ], "l:683": [ { "axis0": [ 0, 1 ], "axis1": [ 115, 6 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 115, 6 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 998, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1819, 1 ], "axis1": [ 115, 6 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 115, 6 ], "axis1": [ 998, 1 ], "matrix": "L2.mlp.W_out" } ], "l:684": [ { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 1715, 3 ], "axis1": [ 704, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 316, 3 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 316, 3 ], "matrix": "L5.mlp.W_out" } ], "l:685": [ { "axis0": [ 0, 1 ], "axis1": [ 54, 12 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 54, 12 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 54, 12 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1715, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1070, 1 ], "axis1": [ 54, 12 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1688, 1 ], "axis1": [ 54, 12 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 54, 12 ], "axis1": [ 1715, 3 ], "matrix": "L4.mlp.W_out" } ], "l:687": [ { "axis0": [ 0, 1 ], "axis1": [ 276, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 836, 1 ], "axis1": [ 276, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 836, 1 ], "axis1": [ 277, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 276, 1 ], "axis1": [ 1070, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 277, 1 ], "axis1": [ 1070, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 276, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 277, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1070, 1 ], "matrix": "L3.mlp.W_out" } ], "l:688": [ { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1070, 1 ], "axis1": [ 640, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 1688, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1688, 1 ], "matrix": "L4.mlp.W_out" } ], "l:689": [ { "axis0": [ 0, 1 ], "axis1": [ 1028, 13 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1042, 4 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1047, 3 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1052, 9 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1062, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1064, 3 ], "matrix": "L5.mlp.W_out" } ], "l:69": [ { "axis0": [ 165, 1 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 1084, 1 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 166, 1 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1796, 1 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 40, 1 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 1679, 1 ], "matrix": "L4.attn.W_O" } ], "l:690": [ { "axis0": [ 0, 1 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 38, 2 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 164, 3 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 828, 3 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 832, 2 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 836, 4 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 841, 4 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 983, 3 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 994, 4 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1664, 4 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1792, 4 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 164, 3 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 828, 3 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 832, 2 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 836, 4 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 841, 4 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 983, 3 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 994, 4 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1664, 4 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1792, 4 ], "axis1": [ 217, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 217, 33 ], "axis1": [ 419, 13 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 217, 33 ], "axis1": [ 465, 20 ], "matrix": "L5.mlp.W_out" } ], "l:692": [ { "axis0": [ 0, 1 ], "axis1": [ 172, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 172, 14 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 38, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 164, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 828, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 836, 4 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 841, 4 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 983, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 994, 4 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 4 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1792, 4 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 51, 1 ], "axis1": [ 172, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 172, 14 ], "axis1": [ 38, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 172, 14 ], "axis1": [ 164, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 172, 14 ], "axis1": [ 828, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 172, 14 ], "axis1": [ 832, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 172, 14 ], "axis1": [ 836, 4 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 172, 14 ], "axis1": [ 841, 4 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 172, 14 ], "axis1": [ 983, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 172, 14 ], "axis1": [ 994, 4 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 172, 14 ], "axis1": [ 1664, 4 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 172, 14 ], "axis1": [ 1792, 4 ], "matrix": "L4.mlp.W_out" } ], "l:693": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1078, 1 ], "axis1": [ 64, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 51, 1 ], "matrix": "L4.attn.W_O" } ], "l:694": [ { "axis0": [ 0, 1 ], "axis1": [ 29, 2 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 29, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 1, 33 ], "axis1": [ 30, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 29, 1 ], "axis1": [ 51, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" }, { "axis0": [ 30, 1 ], "axis1": [ 51, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" } ], "l:695": [ { "axis0": [ 0, 1 ], "axis1": [ 292, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 1003, 1 ], "axis1": [ 292, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1003, 1 ], "axis1": [ 293, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 292, 1 ], "axis1": [ 1078, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 293, 1 ], "axis1": [ 1078, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1078, 1 ], "matrix": "L3.mlp.W_out" } ], "l:696": [ { "axis0": [ 0, 1 ], "axis1": [ 129, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 129, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1003, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1795, 1 ], "axis1": [ 129, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1809, 1 ], "axis1": [ 129, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 129, 2 ], "axis1": [ 1003, 1 ], "matrix": "L2.mlp.W_out" } ], "l:698": [ { "axis0": [ 0, 1 ], "axis1": [ 24, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 24, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1809, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 51, 1 ], "axis1": [ 24, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 24, 2 ], "axis1": [ 1809, 1 ], "matrix": "L1.mlp.W_out" } ], "l:699": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 1 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1795, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 1328, 33 ], "axis1": [ 5, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 5, 1 ], "axis1": [ 1795, 1 ], "matrix": "L1.mlp.W_out" } ], "l:7": [ { "axis0": [ 0, 1 ], "axis1": [ 4, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 4, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 41, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 33 ], "axis1": [ 4, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 4, 1 ], "axis1": [ 41, 1 ], "matrix": "L0.mlp.W_out" } ], "l:700": [ { "axis0": [ 140, 1 ], "axis1": [ 1088, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 140, 1 ], "axis1": [ 1088, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1088, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1088, 64 ], "axis1": [ 1328, 33 ], "matrix": "L1.attn.W_O" } ], "l:701": [ { "axis0": [ 0, 1 ], "axis1": [ 140, 1 ], "matrix": "L0.mlp.W_out" } ], "l:702": [ { "axis0": [ 0, 1 ], "axis1": [ 84, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 84, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 213, 33 ], "axis1": [ 84, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1332, 1 ], "axis1": [ 84, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 213, 33 ], "axis1": [ 84, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1332, 1 ], "axis1": [ 84, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 84, 33 ], "axis1": [ 582, 33 ], "matrix": "L8.mlp.W_out" } ], "l:704": [ { "axis0": [ 0, 1 ], "axis1": [ 902, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 902, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1332, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1044, 2 ], "axis1": [ 902, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 902, 2 ], "axis1": [ 1332, 1 ], "matrix": "L4.mlp.W_out" } ], "l:706": [ { "axis0": [ 0, 1 ], "axis1": [ 224, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 224, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 224, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1044, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 224, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 224, 2 ], "axis1": [ 1044, 1 ], "matrix": "L3.mlp.W_out" } ], "l:707": [ { "axis0": [ 0, 1 ], "axis1": [ 226, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 226, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 226, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1045, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 226, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 226, 2 ], "axis1": [ 1045, 1 ], "matrix": "L3.mlp.W_out" } ], "l:708": [ { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 174, 64 ], "axis1": [ 320, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 213, 33 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 238, 2 ], "axis1": [ 384, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 485, 33 ], "axis1": [ 384, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 213, 33 ], "matrix": "L8.attn.W_O" } ], "l:71": [ { "axis0": [ 0, 1 ], "axis1": [ 1084, 1 ], "matrix": "L3.mlp.W_out" } ], "l:710": [ { "axis0": [ 0, 1 ], "axis1": [ 106, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 106, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1256, 33 ], "axis1": [ 106, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 106, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1256, 33 ], "axis1": [ 106, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 106, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 106, 33 ], "axis1": [ 174, 33 ], "matrix": "L7.mlp.W_out" } ], "l:712": [ { "axis0": [ 0, 1 ], "axis1": [ 136, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 136, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 333, 18 ], "axis1": [ 136, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 352, 1 ], "axis1": [ 136, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 617, 15 ], "axis1": [ 136, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 935, 33 ], "axis1": [ 136, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 333, 18 ], "axis1": [ 136, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 352, 1 ], "axis1": [ 136, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 617, 15 ], "axis1": [ 136, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 935, 33 ], "axis1": [ 136, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 136, 66 ], "axis1": [ 1256, 33 ], "matrix": "L6.mlp.W_out" } ], "l:714": [ { "axis0": [ 0, 1 ], "axis1": [ 145, 6 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 145, 6 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 352, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 1074, 1 ], "axis1": [ 145, 6 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 145, 6 ], "axis1": [ 352, 1 ], "matrix": "L5.mlp.W_out" } ], "l:715": [ { "axis0": [ 0, 1 ], "axis1": [ 284, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 120, 4 ], "axis1": [ 284, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1642, 1 ], "axis1": [ 284, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1649, 4 ], "axis1": [ 284, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1698, 1 ], "axis1": [ 284, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1707, 1 ], "axis1": [ 284, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1730, 22 ], "axis1": [ 284, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 120, 4 ], "axis1": [ 285, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1642, 1 ], "axis1": [ 285, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1649, 4 ], "axis1": [ 285, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1698, 1 ], "axis1": [ 285, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1707, 1 ], "axis1": [ 285, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1730, 22 ], "axis1": [ 285, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 284, 1 ], "axis1": [ 1074, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 285, 1 ], "axis1": [ 1074, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" } ], "l:716": [ { "axis0": [ 1839, 1 ], "axis1": [ 192, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 1839, 1 ], "axis1": [ 192, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 192, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 120, 4 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1642, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1649, 4 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1698, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1707, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 1730, 22 ], "matrix": "L3.attn.W_O" } ], "l:717": [ { "axis0": [ 0, 1 ], "axis1": [ 1839, 1 ], "matrix": "L1.mlp.W_out" } ], "l:718": [ { "axis0": [ 340, 4 ], "axis1": [ 448, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 312, 4 ], "axis1": [ 448, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 448, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 333, 18 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 617, 15 ], "matrix": "L6.attn.W_O" } ], "l:719": [ { "axis0": [ 0, 1 ], "axis1": [ 95, 14 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 95, 14 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 340, 4 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 1074, 1 ], "axis1": [ 95, 14 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 95, 14 ], "axis1": [ 340, 4 ], "matrix": "L5.mlp.W_out" } ], "l:72": [ { "axis0": [ 0, 1 ], "axis1": [ 165, 1 ], "matrix": "L0.mlp.W_out" } ], "l:720": [ { "axis0": [ 0, 1 ], "axis1": [ 139, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 139, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1458, 3 ], "axis1": [ 139, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1462, 30 ], "axis1": [ 139, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 139, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1458, 3 ], "axis1": [ 139, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1462, 30 ], "axis1": [ 139, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 139, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 139, 33 ], "axis1": [ 207, 33 ], "matrix": "L7.mlp.W_out" } ], "l:722": [ { "axis0": [ 0, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 848, 2 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 851, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 853, 2 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 856, 4 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 862, 15 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 878, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 885, 2 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 929, 6 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 968, 14 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 986, 2 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 993, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 999, 4 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1004, 4 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1014, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1018, 2 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1022, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1024, 4 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1724, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 848, 2 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 851, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 853, 2 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 856, 4 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 862, 15 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 878, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 885, 2 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 929, 6 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 968, 14 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 986, 2 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 993, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 999, 4 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1004, 4 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1014, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1018, 2 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1022, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1024, 4 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1724, 1 ], "axis1": [ 400, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 400, 66 ], "axis1": [ 1458, 3 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 400, 66 ], "axis1": [ 1462, 30 ], "matrix": "L6.mlp.W_out" } ], "l:724": [ { "axis0": [ 0, 1 ], "axis1": [ 108, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 108, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1724, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 993, 1 ], "axis1": [ 108, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 108, 2 ], "axis1": [ 1724, 1 ], "matrix": "L4.mlp.W_out" } ], "l:725": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 846, 1 ], "axis1": [ 0, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 993, 1 ], "matrix": "L4.attn.W_O" } ], "l:726": [ { "axis0": [ 0, 1 ], "axis1": [ 94, 3 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 94, 3 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 993, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 138, 1 ], "axis1": [ 94, 3 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1823, 1 ], "axis1": [ 94, 3 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 94, 3 ], "axis1": [ 993, 1 ], "matrix": "L2.mlp.W_out" } ], "l:728": [ { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 965, 33 ], "axis1": [ 832, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 1823, 1 ], "matrix": "L2.attn.W_O" } ], "l:729": [ { "axis0": [ 125, 1 ], "axis1": [ 384, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 125, 1 ], "axis1": [ 384, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 384, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 965, 33 ], "matrix": "L1.attn.W_O" } ], "l:730": [ { "axis0": [ 0, 1 ], "axis1": [ 125, 1 ], "matrix": "L0.mlp.W_out" } ], "l:731": [ { "axis0": [ 0, 1 ], "axis1": [ 138, 1 ], "matrix": "L0.mlp.W_out" } ], "l:732": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 6 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 6 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 846, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 1011, 1 ], "axis1": [ 3, 6 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 3, 6 ], "axis1": [ 846, 1 ], "matrix": "L3.mlp.W_out" } ], "l:733": [ { "axis0": [ 0, 1 ], "axis1": [ 143, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 83, 1 ], "axis1": [ 143, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1644, 5 ], "axis1": [ 143, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1694, 1 ], "axis1": [ 143, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1697, 1 ], "axis1": [ 143, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1708, 25 ], "axis1": [ 143, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 83, 1 ], "axis1": [ 144, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1644, 5 ], "axis1": [ 144, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1694, 1 ], "axis1": [ 144, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1697, 1 ], "axis1": [ 144, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1708, 25 ], "axis1": [ 144, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1011, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 144, 1 ], "axis1": [ 1011, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" } ], "l:734": [ { "axis0": [ 130, 1 ], "axis1": [ 256, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 130, 1 ], "axis1": [ 256, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 256, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 83, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 1644, 5 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 1694, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 1697, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 1708, 25 ], "matrix": "L2.attn.W_O" } ], "l:735": [ { "axis0": [ 0, 1 ], "axis1": [ 130, 1 ], "matrix": "L0.mlp.W_out" } ], "l:736": [ { "axis0": [ 0, 1 ], "axis1": [ 968, 14 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 986, 2 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 993, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 999, 4 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1004, 4 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1014, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1018, 2 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1022, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 4 ], "matrix": "L5.mlp.W_out" } ], "l:737": [ { "axis0": [ 1647, 1 ], "axis1": [ 704, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 1722, 2 ], "axis1": [ 704, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 316, 3 ], "axis1": [ 704, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 704, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 848, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 851, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 853, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 856, 4 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 862, 15 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 878, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 885, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 929, 6 ], "matrix": "L6.attn.W_O" } ], "l:738": [ { "axis0": [ 0, 1 ], "axis1": [ 90, 18 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 90, 18 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 993, 1 ], "axis1": [ 90, 18 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 90, 18 ], "axis1": [ 1647, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 90, 18 ], "axis1": [ 1722, 2 ], "matrix": "L4.mlp.W_out" } ], "l:739": [ { "axis0": [ 0, 1 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 71, 1 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 150, 14 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 831, 1 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1658, 4 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1778, 12 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1791, 1 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 71, 1 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 150, 14 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 831, 1 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1658, 4 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1778, 12 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1791, 1 ], "axis1": [ 250, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 250, 33 ], "axis1": [ 485, 33 ], "matrix": "L5.mlp.W_out" } ], "l:74": [ { "axis0": [ 0, 1 ], "axis1": [ 166, 1 ], "matrix": "L0.mlp.W_out" } ], "l:741": [ { "axis0": [ 0, 1 ], "axis1": [ 158, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 158, 14 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 71, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 150, 14 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 831, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1658, 4 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1778, 12 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1791, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 52, 1 ], "axis1": [ 158, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 158, 14 ], "axis1": [ 71, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 158, 14 ], "axis1": [ 150, 14 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 158, 14 ], "axis1": [ 831, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 158, 14 ], "axis1": [ 1658, 4 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 158, 14 ], "axis1": [ 1778, 12 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 158, 14 ], "axis1": [ 1791, 1 ], "matrix": "L4.mlp.W_out" } ], "l:742": [ { "axis0": [ 0, 1 ], "axis1": [ 302, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 841, 1 ], "axis1": [ 302, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 841, 1 ], "axis1": [ 303, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 52, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 303, 1 ], "axis1": [ 52, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 302, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 303, 1 ], "matrix": "L3.mlp.W_in" } ], "l:743": [ { "axis0": [ 0, 1 ], "axis1": [ 31, 2 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 31, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 1, 33 ], "axis1": [ 32, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 31, 1 ], "axis1": [ 52, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" }, { "axis0": [ 32, 1 ], "axis1": [ 52, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" } ], "l:744": [ { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1004, 1 ], "axis1": [ 768, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 841, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 841, 1 ], "matrix": "L3.mlp.W_out" } ], "l:745": [ { "axis0": [ 0, 1 ], "axis1": [ 131, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 131, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1004, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1796, 1 ], "axis1": [ 131, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1810, 1 ], "axis1": [ 131, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 131, 2 ], "axis1": [ 1004, 1 ], "matrix": "L2.mlp.W_out" } ], "l:747": [ { "axis0": [ 0, 1 ], "axis1": [ 26, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 26, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1810, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 52, 1 ], "axis1": [ 26, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 26, 2 ], "axis1": [ 1810, 1 ], "matrix": "L1.mlp.W_out" } ], "l:748": [ { "axis0": [ 0, 1 ], "axis1": [ 6, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 6, 1 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1796, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 1361, 33 ], "axis1": [ 6, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 6, 1 ], "axis1": [ 1796, 1 ], "matrix": "L1.mlp.W_out" } ], "l:749": [ { "axis0": [ 141, 1 ], "axis1": [ 1152, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 141, 1 ], "axis1": [ 1152, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1152, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1152, 64 ], "axis1": [ 1361, 33 ], "matrix": "L1.attn.W_O" } ], "l:75": [ { "axis0": [ 0, 1 ], "axis1": [ 300, 33 ], "matrix": "L0.mlp.W_out" } ], "l:750": [ { "axis0": [ 0, 1 ], "axis1": [ 141, 1 ], "matrix": "L0.mlp.W_out" } ], "l:751": [ { "axis0": [ 0, 1 ], "axis1": [ 117, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 117, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 246, 33 ], "axis1": [ 117, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1461, 1 ], "axis1": [ 117, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 246, 33 ], "axis1": [ 117, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1461, 1 ], "axis1": [ 117, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 117, 33 ], "axis1": [ 615, 33 ], "matrix": "L8.mlp.W_out" } ], "l:753": [ { "axis0": [ 0, 1 ], "axis1": [ 904, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 904, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1461, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1047, 1 ], "axis1": [ 904, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1836, 1 ], "axis1": [ 904, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 904, 2 ], "axis1": [ 1461, 1 ], "matrix": "L4.mlp.W_out" } ], "l:755": [ { "axis0": [ 0, 1 ], "axis1": [ 228, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 228, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 228, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1836, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 228, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 228, 2 ], "axis1": [ 1836, 1 ], "matrix": "L3.mlp.W_out" } ], "l:756": [ { "axis0": [ 0, 1 ], "axis1": [ 230, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 230, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 230, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1047, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 230, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 230, 2 ], "axis1": [ 1047, 1 ], "matrix": "L3.mlp.W_out" } ], "l:757": [ { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 240, 64 ], "axis1": [ 448, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 246, 33 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 304, 2 ], "axis1": [ 512, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 518, 13 ], "axis1": [ 512, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 597, 20 ], "axis1": [ 512, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 246, 33 ], "matrix": "L8.attn.W_O" } ], "l:759": [ { "axis0": [ 0, 1 ], "axis1": [ 172, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 172, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1289, 25 ], "axis1": [ 172, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1380, 2 ], "axis1": [ 172, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1383, 3 ], "axis1": [ 172, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1387, 3 ], "axis1": [ 172, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 172, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1289, 25 ], "axis1": [ 172, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1380, 2 ], "axis1": [ 172, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1383, 3 ], "axis1": [ 172, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1387, 3 ], "axis1": [ 172, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 172, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 172, 33 ], "axis1": [ 240, 33 ], "matrix": "L7.mlp.W_out" } ], "l:76": [ { "axis0": [ 0, 1 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 37, 2 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 61, 6 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 72, 3 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 76, 2 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 79, 4 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 94, 12 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1023, 1 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 37, 2 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 52, 1 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 61, 6 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 69, 1 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 72, 3 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 76, 2 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 79, 4 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 94, 12 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1023, 1 ], "axis1": [ 788, 33 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 788, 33 ], "axis1": [ 1581, 6 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 788, 33 ], "axis1": [ 1986, 26 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 788, 33 ], "axis1": [ 2045, 1 ], "matrix": "L14.mlp.W_out" } ], "l:761": [ { "axis0": [ 0, 1 ], "axis1": [ 202, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 202, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 632, 33 ], "axis1": [ 202, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 887, 1 ], "axis1": [ 202, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 935, 33 ], "axis1": [ 202, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 632, 33 ], "axis1": [ 202, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 887, 1 ], "axis1": [ 202, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 935, 33 ], "axis1": [ 202, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 202, 66 ], "axis1": [ 1289, 25 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 202, 66 ], "axis1": [ 1380, 2 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 202, 66 ], "axis1": [ 1383, 3 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 202, 66 ], "axis1": [ 1387, 3 ], "matrix": "L6.mlp.W_out" } ], "l:763": [ { "axis0": [ 0, 1 ], "axis1": [ 122, 6 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 122, 6 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 887, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 1012, 1 ], "axis1": [ 122, 6 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 122, 6 ], "axis1": [ 887, 1 ], "matrix": "L3.mlp.W_out" } ], "l:764": [ { "axis0": [ 0, 1 ], "axis1": [ 145, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 120, 5 ], "axis1": [ 145, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1642, 1 ], "axis1": [ 145, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1649, 5 ], "axis1": [ 145, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1698, 1 ], "axis1": [ 145, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1707, 1 ], "axis1": [ 145, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1733, 20 ], "axis1": [ 145, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 120, 5 ], "axis1": [ 146, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1642, 1 ], "axis1": [ 146, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1649, 5 ], "axis1": [ 146, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1698, 1 ], "axis1": [ 146, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1707, 1 ], "axis1": [ 146, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1733, 20 ], "axis1": [ 146, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1012, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 146, 1 ], "axis1": [ 1012, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" } ], "l:765": [ { "axis0": [ 1840, 1 ], "axis1": [ 320, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 1840, 1 ], "axis1": [ 320, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 320, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 120, 5 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 1642, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 1649, 5 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 1698, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 1707, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 1733, 20 ], "matrix": "L2.attn.W_O" } ], "l:766": [ { "axis0": [ 0, 1 ], "axis1": [ 1840, 1 ], "matrix": "L1.mlp.W_out" } ], "l:767": [ { "axis0": [ 878, 1 ], "axis1": [ 512, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 885, 2 ], "axis1": [ 512, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 1663, 1 ], "axis1": [ 512, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 312, 4 ], "axis1": [ 512, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 512, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 632, 33 ], "matrix": "L6.attn.W_O" } ], "l:768": [ { "axis0": [ 0, 1 ], "axis1": [ 87, 9 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 87, 9 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 878, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 885, 2 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1663, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 1012, 1 ], "axis1": [ 87, 9 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 87, 9 ], "axis1": [ 878, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 87, 9 ], "axis1": [ 885, 2 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 87, 9 ], "axis1": [ 1663, 1 ], "matrix": "L3.mlp.W_out" } ], "l:769": [ { "axis0": [ 0, 1 ], "axis1": [ 205, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 205, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1493, 33 ], "axis1": [ 205, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 205, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1493, 33 ], "axis1": [ 205, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 205, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 205, 33 ], "axis1": [ 273, 33 ], "matrix": "L7.mlp.W_out" } ], "l:771": [ { "axis0": [ 0, 1 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 881, 1 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 968, 14 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 983, 5 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 993, 15 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1009, 11 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1021, 2 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1024, 4 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1050, 1 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1068, 1 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1073, 5 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1079, 2 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1082, 3 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1114, 3 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 881, 1 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 968, 14 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 983, 5 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 993, 15 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1009, 11 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1021, 2 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1024, 4 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1050, 1 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1068, 1 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1073, 5 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1079, 2 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1082, 3 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1114, 3 ], "axis1": [ 466, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 466, 66 ], "axis1": [ 1493, 33 ], "matrix": "L6.mlp.W_out" } ], "l:773": [ { "axis0": [ 0, 1 ], "axis1": [ 96, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 96, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 881, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 994, 1 ], "axis1": [ 96, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 96, 2 ], "axis1": [ 881, 1 ], "matrix": "L3.mlp.W_out" } ], "l:774": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 997, 1 ], "axis1": [ 0, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 994, 1 ], "matrix": "L3.attn.W_O" } ], "l:775": [ { "axis0": [ 0, 1 ], "axis1": [ 97, 3 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 97, 3 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 994, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 981, 1 ], "axis1": [ 97, 3 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1838, 1 ], "axis1": [ 97, 3 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 97, 3 ], "axis1": [ 994, 1 ], "matrix": "L2.mlp.W_out" } ], "l:777": [ { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 998, 33 ], "axis1": [ 896, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 981, 1 ], "matrix": "L2.attn.W_O" } ], "l:778": [ { "axis0": [ 126, 1 ], "axis1": [ 448, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 126, 1 ], "axis1": [ 448, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 448, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 998, 33 ], "matrix": "L1.attn.W_O" } ], "l:779": [ { "axis0": [ 0, 1 ], "axis1": [ 126, 1 ], "matrix": "L0.mlp.W_out" } ], "l:78": [ { "axis0": [ 0, 1 ], "axis1": [ 876, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 876, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1023, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1006, 2 ], "axis1": [ 876, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 876, 2 ], "axis1": [ 1023, 1 ], "matrix": "L4.mlp.W_out" } ], "l:780": [ { "axis0": [ 0, 1 ], "axis1": [ 1838, 1 ], "matrix": "L1.mlp.W_out" } ], "l:781": [ { "axis0": [ 0, 1 ], "axis1": [ 109, 6 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 109, 6 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 997, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1820, 1 ], "axis1": [ 109, 6 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 109, 6 ], "axis1": [ 997, 1 ], "matrix": "L2.mlp.W_out" } ], "l:782": [ { "axis0": [ 0, 1 ], "axis1": [ 46, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1130, 33 ], "axis1": [ 46, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1130, 33 ], "axis1": [ 47, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 46, 1 ], "axis1": [ 1820, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 47, 1 ], "axis1": [ 1820, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:783": [ { "axis0": [ 131, 1 ], "axis1": [ 704, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 131, 1 ], "axis1": [ 704, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 704, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 1130, 33 ], "matrix": "L1.attn.W_O" } ], "l:784": [ { "axis0": [ 0, 1 ], "axis1": [ 131, 1 ], "matrix": "L0.mlp.W_out" } ], "l:785": [ { "axis0": [ 874, 3 ], "axis1": [ 768, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 316, 3 ], "axis1": [ 768, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 768, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 983, 3 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 994, 5 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1003, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1009, 5 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1015, 3 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1021, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1050, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1068, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1073, 5 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1079, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1082, 3 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1114, 3 ], "matrix": "L6.attn.W_O" } ], "l:786": [ { "axis0": [ 0, 1 ], "axis1": [ 69, 18 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 18 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 994, 1 ], "axis1": [ 69, 18 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 69, 18 ], "axis1": [ 874, 3 ], "matrix": "L3.mlp.W_out" } ], "l:787": [ { "axis0": [ 0, 1 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 66, 1 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 135, 2 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 138, 12 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1656, 2 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1699, 1 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1763, 15 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 135, 2 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 138, 12 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1656, 2 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1699, 1 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1763, 15 ], "axis1": [ 283, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 283, 33 ], "axis1": [ 518, 13 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 283, 33 ], "axis1": [ 597, 20 ], "matrix": "L5.mlp.W_out" } ], "l:789": [ { "axis0": [ 0, 1 ], "axis1": [ 144, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 144, 14 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 66, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 135, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 138, 12 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1656, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1699, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1763, 15 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 53, 1 ], "axis1": [ 144, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 144, 14 ], "axis1": [ 66, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 144, 14 ], "axis1": [ 135, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 144, 14 ], "axis1": [ 138, 12 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 144, 14 ], "axis1": [ 1656, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 144, 14 ], "axis1": [ 1699, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 144, 14 ], "axis1": [ 1763, 15 ], "matrix": "L4.mlp.W_out" } ], "l:790": [ { "axis0": [ 0, 1 ], "axis1": [ 304, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 842, 1 ], "axis1": [ 304, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 842, 1 ], "axis1": [ 305, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 304, 1 ], "axis1": [ 53, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 305, 1 ], "axis1": [ 53, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 304, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 305, 1 ], "matrix": "L3.mlp.W_in" } ], "l:791": [ { "axis0": [ 0, 1 ], "axis1": [ 33, 2 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 33, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 1, 33 ], "axis1": [ 34, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 53, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" }, { "axis0": [ 34, 1 ], "axis1": [ 53, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" } ], "l:792": [ { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1005, 1 ], "axis1": [ 832, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 842, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 842, 1 ], "matrix": "L3.mlp.W_out" } ], "l:793": [ { "axis0": [ 0, 1 ], "axis1": [ 133, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 133, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1005, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1797, 1 ], "axis1": [ 133, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1811, 1 ], "axis1": [ 133, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 133, 2 ], "axis1": [ 1005, 1 ], "matrix": "L2.mlp.W_out" } ], "l:795": [ { "axis0": [ 0, 1 ], "axis1": [ 28, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 28, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1811, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 53, 1 ], "axis1": [ 28, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 28, 2 ], "axis1": [ 1811, 1 ], "matrix": "L1.mlp.W_out" } ], "l:796": [ { "axis0": [ 0, 1 ], "axis1": [ 7, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 1 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1797, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 1394, 33 ], "axis1": [ 7, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 7, 1 ], "axis1": [ 1797, 1 ], "matrix": "L1.mlp.W_out" } ], "l:797": [ { "axis0": [ 142, 1 ], "axis1": [ 1216, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 142, 1 ], "axis1": [ 1216, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1216, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1216, 64 ], "axis1": [ 1394, 33 ], "matrix": "L1.attn.W_O" } ], "l:798": [ { "axis0": [ 0, 1 ], "axis1": [ 142, 1 ], "matrix": "L0.mlp.W_out" } ], "l:799": [ { "axis0": [ 0, 1 ], "axis1": [ 150, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 150, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 279, 33 ], "axis1": [ 150, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1081, 1 ], "axis1": [ 150, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 279, 33 ], "axis1": [ 150, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1081, 1 ], "axis1": [ 150, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 150, 33 ], "axis1": [ 648, 21 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 150, 33 ], "axis1": [ 702, 12 ], "matrix": "L8.mlp.W_out" } ], "l:80": [ { "axis0": [ 0, 1 ], "axis1": [ 168, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 168, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 168, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1006, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 168, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 168, 2 ], "axis1": [ 1006, 1 ], "matrix": "L3.mlp.W_out" } ], "l:801": [ { "axis0": [ 0, 1 ], "axis1": [ 906, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 906, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1081, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1048, 2 ], "axis1": [ 906, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 906, 2 ], "axis1": [ 1081, 1 ], "matrix": "L4.mlp.W_out" } ], "l:803": [ { "axis0": [ 0, 1 ], "axis1": [ 232, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 232, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 232, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1048, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 232, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 232, 2 ], "axis1": [ 1048, 1 ], "matrix": "L3.mlp.W_out" } ], "l:804": [ { "axis0": [ 0, 1 ], "axis1": [ 234, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 234, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 234, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1049, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 234, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 234, 2 ], "axis1": [ 1049, 1 ], "matrix": "L3.mlp.W_out" } ], "l:805": [ { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 306, 47 ], "axis1": [ 576, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 617, 17 ], "axis1": [ 576, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 279, 33 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 634, 35 ], "axis1": [ 640, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 279, 33 ], "matrix": "L8.attn.W_O" } ], "l:807": [ { "axis0": [ 0, 1 ], "axis1": [ 238, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 238, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1390, 33 ], "axis1": [ 238, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 238, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1390, 33 ], "axis1": [ 238, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 238, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 238, 33 ], "axis1": [ 306, 33 ], "matrix": "L7.mlp.W_out" } ], "l:809": [ { "axis0": [ 0, 1 ], "axis1": [ 268, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 268, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 351, 1 ], "axis1": [ 268, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 665, 33 ], "axis1": [ 268, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 935, 33 ], "axis1": [ 268, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 351, 1 ], "axis1": [ 268, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 665, 33 ], "axis1": [ 268, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 935, 33 ], "axis1": [ 268, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 268, 66 ], "axis1": [ 1390, 33 ], "matrix": "L6.mlp.W_out" } ], "l:81": [ { "axis0": [ 0, 1 ], "axis1": [ 170, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 170, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 170, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1007, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 170, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 170, 2 ], "axis1": [ 1007, 1 ], "matrix": "L3.mlp.W_out" } ], "l:811": [ { "axis0": [ 0, 1 ], "axis1": [ 141, 4 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 141, 4 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 351, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 1075, 1 ], "axis1": [ 141, 4 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 141, 4 ], "axis1": [ 351, 1 ], "matrix": "L5.mlp.W_out" } ], "l:812": [ { "axis0": [ 0, 1 ], "axis1": [ 286, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 70, 1 ], "axis1": [ 286, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 124, 13 ], "axis1": [ 286, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 286, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1653, 3 ], "axis1": [ 286, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1752, 15 ], "axis1": [ 286, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 70, 1 ], "axis1": [ 287, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 124, 13 ], "axis1": [ 287, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 287, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1653, 3 ], "axis1": [ 287, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1752, 15 ], "axis1": [ 287, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 286, 1 ], "axis1": [ 1075, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 287, 1 ], "axis1": [ 1075, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" } ], "l:813": [ { "axis0": [ 146, 1 ], "axis1": [ 256, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 146, 1 ], "axis1": [ 256, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 256, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 70, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 124, 13 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 138, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 1653, 3 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 1752, 15 ], "matrix": "L3.attn.W_O" } ], "l:814": [ { "axis0": [ 0, 1 ], "axis1": [ 146, 1 ], "matrix": "L0.mlp.W_out" } ], "l:815": [ { "axis0": [ 336, 4 ], "axis1": [ 576, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 312, 4 ], "axis1": [ 576, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 576, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 665, 33 ], "matrix": "L6.attn.W_O" } ], "l:816": [ { "axis0": [ 0, 1 ], "axis1": [ 91, 4 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 91, 4 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 336, 4 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 1075, 1 ], "axis1": [ 91, 4 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 91, 4 ], "axis1": [ 336, 4 ], "matrix": "L5.mlp.W_out" } ], "l:817": [ { "axis0": [ 0, 1 ], "axis1": [ 271, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 271, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1526, 9 ], "axis1": [ 271, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1536, 22 ], "axis1": [ 271, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1559, 2 ], "axis1": [ 271, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 271, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1526, 9 ], "axis1": [ 271, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1536, 22 ], "axis1": [ 271, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1559, 2 ], "axis1": [ 271, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 271, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 271, 33 ], "axis1": [ 339, 14 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 271, 33 ], "axis1": [ 617, 19 ], "matrix": "L7.mlp.W_out" } ], "l:819": [ { "axis0": [ 0, 1 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 882, 1 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 968, 14 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 986, 2 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 993, 1 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 999, 4 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1004, 4 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1014, 1 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1018, 2 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1022, 1 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1024, 4 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1117, 33 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 882, 1 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 968, 14 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 986, 2 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 993, 1 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 999, 4 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1004, 4 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1014, 1 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1018, 2 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1022, 1 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1024, 4 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1117, 33 ], "axis1": [ 532, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 532, 66 ], "axis1": [ 1526, 9 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 532, 66 ], "axis1": [ 1536, 22 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 532, 66 ], "axis1": [ 1559, 2 ], "matrix": "L6.mlp.W_out" } ], "l:82": [ { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 35, 1 ], "axis1": [ 384, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 228, 1 ], "axis1": [ 384, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 327, 1 ], "axis1": [ 384, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 869, 8 ], "axis1": [ 384, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 878, 1 ], "axis1": [ 384, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 883, 5 ], "axis1": [ 384, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 965, 1 ], "axis1": [ 384, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 972, 10 ], "axis1": [ 384, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 984, 2 ], "axis1": [ 384, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1006, 1 ], "axis1": [ 384, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1487, 33 ], "axis1": [ 384, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 35, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 37, 2 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 40, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 52, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 61, 6 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 69, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 72, 3 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 76, 2 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 79, 4 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 94, 12 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 444, 33 ], "axis1": [ 448, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1007, 1 ], "axis1": [ 448, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1009, 1 ], "axis1": [ 448, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 35, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 37, 2 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 40, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 52, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 61, 6 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 69, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 72, 3 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 76, 2 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 79, 4 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 94, 12 ], "matrix": "L14.attn.W_O" } ], "l:821": [ { "axis0": [ 0, 1 ], "axis1": [ 98, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 98, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 882, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 995, 1 ], "axis1": [ 98, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 98, 2 ], "axis1": [ 882, 1 ], "matrix": "L3.mlp.W_out" } ], "l:822": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 996, 1 ], "axis1": [ 64, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 995, 1 ], "matrix": "L3.attn.W_O" } ], "l:823": [ { "axis0": [ 0, 1 ], "axis1": [ 100, 3 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 100, 3 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 995, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 139, 1 ], "axis1": [ 100, 3 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1817, 1 ], "axis1": [ 100, 3 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 100, 3 ], "axis1": [ 995, 1 ], "matrix": "L2.mlp.W_out" } ], "l:825": [ { "axis0": [ 0, 1 ], "axis1": [ 40, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1031, 33 ], "axis1": [ 40, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1031, 33 ], "axis1": [ 41, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 1817, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 41, 1 ], "axis1": [ 1817, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:826": [ { "axis0": [ 127, 1 ], "axis1": [ 512, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 127, 1 ], "axis1": [ 512, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 512, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 1031, 33 ], "matrix": "L1.attn.W_O" } ], "l:827": [ { "axis0": [ 0, 1 ], "axis1": [ 127, 1 ], "matrix": "L0.mlp.W_out" } ], "l:828": [ { "axis0": [ 0, 1 ], "axis1": [ 139, 1 ], "matrix": "L0.mlp.W_out" } ], "l:829": [ { "axis0": [ 0, 1 ], "axis1": [ 103, 6 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 103, 6 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 996, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1821, 1 ], "axis1": [ 103, 6 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 103, 6 ], "axis1": [ 996, 1 ], "matrix": "L2.mlp.W_out" } ], "l:830": [ { "axis0": [ 0, 1 ], "axis1": [ 48, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 1163, 33 ], "axis1": [ 48, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1163, 33 ], "axis1": [ 49, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 48, 1 ], "axis1": [ 1821, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 49, 1 ], "axis1": [ 1821, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" } ], "l:831": [ { "axis0": [ 132, 1 ], "axis1": [ 768, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 132, 1 ], "axis1": [ 768, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 768, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 1163, 33 ], "matrix": "L1.attn.W_O" } ], "l:832": [ { "axis0": [ 0, 1 ], "axis1": [ 132, 1 ], "matrix": "L0.mlp.W_out" } ], "l:833": [ { "axis0": [ 871, 3 ], "axis1": [ 832, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 316, 3 ], "axis1": [ 832, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 832, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 1117, 33 ], "matrix": "L6.attn.W_O" } ], "l:834": [ { "axis0": [ 0, 1 ], "axis1": [ 51, 18 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 51, 18 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 995, 1 ], "axis1": [ 51, 18 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 51, 18 ], "axis1": [ 871, 3 ], "matrix": "L3.mlp.W_out" } ], "l:835": [ { "axis0": [ 0, 1 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 70, 1 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 121, 14 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1642, 1 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1653, 3 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1698, 1 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1750, 13 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 70, 1 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 121, 14 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1642, 1 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1653, 3 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1698, 1 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1750, 13 ], "axis1": [ 304, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 304, 33 ], "axis1": [ 636, 33 ], "matrix": "L7.mlp.W_out" } ], "l:837": [ { "axis0": [ 0, 1 ], "axis1": [ 130, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 130, 14 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 70, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 121, 14 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1642, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1653, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1698, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1750, 13 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 54, 1 ], "axis1": [ 130, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 130, 14 ], "axis1": [ 70, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 130, 14 ], "axis1": [ 121, 14 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 130, 14 ], "axis1": [ 1642, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 130, 14 ], "axis1": [ 1653, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 130, 14 ], "axis1": [ 1698, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 130, 14 ], "axis1": [ 1750, 13 ], "matrix": "L4.mlp.W_out" } ], "l:838": [ { "axis0": [ 0, 1 ], "axis1": [ 306, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 843, 1 ], "axis1": [ 306, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 843, 1 ], "axis1": [ 307, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 306, 1 ], "axis1": [ 54, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 307, 1 ], "axis1": [ 54, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 306, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 307, 1 ], "matrix": "L3.mlp.W_in" } ], "l:839": [ { "axis0": [ 0, 1 ], "axis1": [ 35, 2 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 1, 33 ], "axis1": [ 35, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 1, 33 ], "axis1": [ 36, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 54, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" }, { "axis0": [ 36, 1 ], "axis1": [ 54, 1 ], "diagonal": true, "matrix": "L0.mlp.W_out" } ], "l:84": [ { "axis0": [ 0, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 38, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 133, 7 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 188, 4 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 233, 2 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 675, 9 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 966, 6 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1008, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 38, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 66, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 133, 7 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 188, 4 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 233, 2 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 331, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 675, 9 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 966, 6 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 1008, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 876, 33 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 876, 33 ], "axis1": [ 1487, 33 ], "matrix": "L13.mlp.W_out" } ], "l:840": [ { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1006, 1 ], "axis1": [ 896, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 843, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 843, 1 ], "matrix": "L3.mlp.W_out" } ], "l:841": [ { "axis0": [ 0, 1 ], "axis1": [ 135, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 135, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1006, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 1798, 1 ], "axis1": [ 135, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1812, 1 ], "axis1": [ 135, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 135, 2 ], "axis1": [ 1006, 1 ], "matrix": "L2.mlp.W_out" } ], "l:843": [ { "axis0": [ 0, 1 ], "axis1": [ 30, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 30, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1812, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 54, 1 ], "axis1": [ 30, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 30, 2 ], "axis1": [ 1812, 1 ], "matrix": "L1.mlp.W_out" } ], "l:844": [ { "axis0": [ 0, 1 ], "axis1": [ 8, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 8, 1 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1798, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 1427, 33 ], "axis1": [ 8, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 8, 1 ], "axis1": [ 1798, 1 ], "matrix": "L1.mlp.W_out" } ], "l:845": [ { "axis0": [ 143, 1 ], "axis1": [ 1280, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 143, 1 ], "axis1": [ 1280, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1280, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1280, 64 ], "axis1": [ 1427, 33 ], "matrix": "L1.attn.W_O" } ], "l:846": [ { "axis0": [ 0, 1 ], "axis1": [ 143, 1 ], "matrix": "L0.mlp.W_out" } ], "l:847": [ { "axis0": [ 0, 1 ], "axis1": [ 994, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 994, 33 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 716, 1 ], "axis1": [ 994, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1150, 33 ], "axis1": [ 994, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 716, 1 ], "axis1": [ 994, 33 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1150, 33 ], "axis1": [ 994, 33 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 994, 33 ], "axis1": [ 2012, 33 ], "matrix": "L6.mlp.W_out" } ], "l:849": [ { "axis0": [ 0, 1 ], "axis1": [ 415, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 415, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 716, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 1050, 1 ], "axis1": [ 415, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1837, 1 ], "axis1": [ 415, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 415, 2 ], "axis1": [ 716, 1 ], "matrix": "L5.mlp.W_out" } ], "l:851": [ { "axis0": [ 0, 1 ], "axis1": [ 236, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 236, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 236, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1050, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 236, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 236, 2 ], "axis1": [ 1050, 1 ], "matrix": "L3.mlp.W_out" } ], "l:852": [ { "axis0": [ 0, 1 ], "axis1": [ 238, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 238, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 238, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1837, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 238, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 238, 2 ], "axis1": [ 1837, 1 ], "matrix": "L3.mlp.W_out" } ], "l:853": [ { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 617, 64 ], "axis1": [ 896, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 1150, 33 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 681, 35 ], "axis1": [ 960, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 960, 64 ], "axis1": [ 1150, 33 ], "matrix": "L6.attn.W_O" } ], "l:855": [ { "axis0": [ 0, 1 ], "axis1": [ 316, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 316, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 201, 33 ], "axis1": [ 316, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 316, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 201, 33 ], "axis1": [ 316, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 316, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 316, 33 ], "axis1": [ 617, 33 ], "matrix": "L5.mlp.W_out" } ], "l:857": [ { "axis0": [ 0, 1 ], "axis1": [ 201, 33 ], "matrix": "L0.mlp.W_out" } ], "l:858": [ { "axis0": [ 0, 1 ], "axis1": [ 349, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 349, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 74, 1 ], "axis1": [ 349, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1118, 21 ], "axis1": [ 349, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1140, 11 ], "axis1": [ 349, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 349, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 74, 1 ], "axis1": [ 349, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1118, 21 ], "axis1": [ 349, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1140, 11 ], "axis1": [ 349, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 349, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 349, 33 ], "axis1": [ 650, 33 ], "matrix": "L5.mlp.W_out" } ], "l:86": [ { "axis0": [ 0, 1 ], "axis1": [ 93, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 93, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 38, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 40, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 66, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 133, 7 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 188, 4 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 233, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 331, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 675, 9 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 966, 6 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1008, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 201, 1 ], "axis1": [ 93, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 93, 2 ], "axis1": [ 38, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 93, 2 ], "axis1": [ 40, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 93, 2 ], "axis1": [ 66, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 93, 2 ], "axis1": [ 133, 7 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 93, 2 ], "axis1": [ 188, 4 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 93, 2 ], "axis1": [ 233, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 93, 2 ], "axis1": [ 331, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 93, 2 ], "axis1": [ 675, 9 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 93, 2 ], "axis1": [ 966, 6 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 93, 2 ], "axis1": [ 1008, 1 ], "matrix": "L12.mlp.W_out" } ], "l:860": [ { "axis0": [ 0, 1 ], "axis1": [ 74, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1118, 21 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1140, 11 ], "matrix": "L3.mlp.W_out" } ], "l:861": [ { "axis0": [ 0, 1 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 69, 1 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 120, 1 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1648, 5 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1697, 1 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1707, 1 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1726, 24 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 120, 1 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1648, 5 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1697, 1 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1707, 1 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1726, 24 ], "axis1": [ 382, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 382, 33 ], "axis1": [ 683, 33 ], "matrix": "L5.mlp.W_out" } ], "l:863": [ { "axis0": [ 0, 1 ], "axis1": [ 116, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 116, 14 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 120, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1648, 5 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1697, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1707, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1726, 24 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 37, 1 ], "axis1": [ 116, 14 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 116, 14 ], "axis1": [ 69, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 116, 14 ], "axis1": [ 120, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 116, 14 ], "axis1": [ 1648, 5 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 116, 14 ], "axis1": [ 1697, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 116, 14 ], "axis1": [ 1707, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 116, 14 ], "axis1": [ 1726, 24 ], "matrix": "L4.mlp.W_out" } ], "l:864": [ { "axis0": [ 0, 1 ], "axis1": [ 308, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 844, 1 ], "axis1": [ 308, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 844, 1 ], "axis1": [ 309, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 308, 1 ], "axis1": [ 37, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 309, 1 ], "axis1": [ 37, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 308, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 309, 1 ], "matrix": "L3.mlp.W_in" } ], "l:865": [ { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L0.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L0.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 192, 64 ], "matrix": "L0.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 37, 1 ], "matrix": "L0.attn.W_O" } ], "l:866": [ { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1007, 1 ], "axis1": [ 960, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 960, 64 ], "axis1": [ 844, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 844, 1 ], "matrix": "L3.mlp.W_out" } ], "l:867": [ { "axis0": [ 0, 1 ], "axis1": [ 137, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 137, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1007, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 48, 1 ], "axis1": [ 137, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1799, 1 ], "axis1": [ 137, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 1007, 1 ], "matrix": "L2.mlp.W_out" } ], "l:869": [ { "axis0": [ 0, 1 ], "axis1": [ 23, 2 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 23, 2 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 48, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 37, 1 ], "axis1": [ 23, 2 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 23, 2 ], "axis1": [ 48, 1 ], "matrix": "L0.mlp.W_out" } ], "l:87": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 192, 1 ], "axis1": [ 128, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 201, 1 ], "matrix": "L12.attn.W_O" } ], "l:870": [ { "axis0": [ 0, 1 ], "axis1": [ 9, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 1 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1799, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 1460, 33 ], "axis1": [ 9, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 9, 1 ], "axis1": [ 1799, 1 ], "matrix": "L1.mlp.W_out" } ], "l:871": [ { "axis0": [ 144, 1 ], "axis1": [ 1344, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 144, 1 ], "axis1": [ 1344, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 1344, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 1344, 64 ], "axis1": [ 1460, 33 ], "matrix": "L1.attn.W_O" } ], "l:872": [ { "axis0": [ 0, 1 ], "axis1": [ 144, 1 ], "matrix": "L0.mlp.W_out" } ], "l:873": [ { "axis0": [ 0, 1 ], "axis1": [ 483, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 483, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 246, 33 ], "axis1": [ 483, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1482, 1 ], "axis1": [ 483, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 246, 33 ], "axis1": [ 483, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1482, 1 ], "axis1": [ 483, 33 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 483, 33 ], "axis1": [ 816, 12 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 483, 33 ], "axis1": [ 834, 2 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 483, 33 ], "axis1": [ 840, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 483, 33 ], "axis1": [ 845, 2 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 483, 33 ], "axis1": [ 850, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 483, 33 ], "axis1": [ 852, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 483, 33 ], "axis1": [ 855, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 483, 33 ], "axis1": [ 860, 2 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 483, 33 ], "axis1": [ 877, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 483, 33 ], "axis1": [ 879, 2 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 483, 33 ], "axis1": [ 888, 8 ], "matrix": "L5.mlp.W_out" } ], "l:875": [ { "axis0": [ 0, 1 ], "axis1": [ 908, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 908, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1482, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1052, 2 ], "axis1": [ 908, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 908, 2 ], "axis1": [ 1482, 1 ], "matrix": "L4.mlp.W_out" } ], "l:877": [ { "axis0": [ 0, 1 ], "axis1": [ 240, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 240, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 240, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1052, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 240, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 240, 2 ], "axis1": [ 1052, 1 ], "matrix": "L3.mlp.W_out" } ], "l:878": [ { "axis0": [ 0, 1 ], "axis1": [ 242, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 242, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 242, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1053, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 242, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 242, 2 ], "axis1": [ 1053, 1 ], "matrix": "L3.mlp.W_out" } ], "l:879": [ { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 61, 1 ], "axis1": [ 384, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1479, 3 ], "axis1": [ 384, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1483, 9 ], "axis1": [ 384, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1493, 42 ], "axis1": [ 384, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1536, 7 ], "axis1": [ 384, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1687, 1 ], "axis1": [ 384, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1705, 1 ], "axis1": [ 384, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 246, 33 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 62, 1 ], "axis1": [ 448, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1543, 15 ], "axis1": [ 448, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1559, 19 ], "axis1": [ 448, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 246, 33 ], "matrix": "L5.attn.W_O" } ], "l:88": [ { "axis0": [ 0, 1 ], "axis1": [ 1877, 2 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 146, 6 ], "axis1": [ 1877, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 158, 7 ], "axis1": [ 1877, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 168, 1 ], "axis1": [ 1877, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 171, 1 ], "axis1": [ 1877, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 174, 1 ], "axis1": [ 1877, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 177, 8 ], "axis1": [ 1877, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 186, 6 ], "axis1": [ 1877, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 195, 1 ], "axis1": [ 1877, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 1877, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1877, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 146, 6 ], "axis1": [ 1878, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 158, 7 ], "axis1": [ 1878, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 168, 1 ], "axis1": [ 1878, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 171, 1 ], "axis1": [ 1878, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 174, 1 ], "axis1": [ 1878, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 177, 8 ], "axis1": [ 1878, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 186, 6 ], "axis1": [ 1878, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 195, 1 ], "axis1": [ 1878, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 1878, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1878, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1877, 1 ], "axis1": [ 201, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1878, 1 ], "axis1": [ 201, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" } ], "l:881": [ { "axis0": [ 0, 1 ], "axis1": [ 478, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 478, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 399, 33 ], "axis1": [ 478, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 478, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 399, 33 ], "axis1": [ 478, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 478, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 478, 33 ], "axis1": [ 1479, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 478, 33 ], "axis1": [ 1483, 9 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 478, 33 ], "axis1": [ 1493, 19 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 478, 33 ], "axis1": [ 1687, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 478, 33 ], "axis1": [ 1705, 1 ], "matrix": "L4.mlp.W_out" } ], "l:883": [ { "axis0": [ 0, 1 ], "axis1": [ 399, 33 ], "matrix": "L0.mlp.W_out" } ], "l:884": [ { "axis0": [ 0, 1 ], "axis1": [ 511, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 511, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 630, 33 ], "axis1": [ 511, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 511, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 630, 33 ], "axis1": [ 511, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 511, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 511, 33 ], "axis1": [ 61, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 511, 33 ], "axis1": [ 1512, 23 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 511, 33 ], "axis1": [ 1536, 9 ], "matrix": "L4.mlp.W_out" } ], "l:886": [ { "axis0": [ 0, 1 ], "axis1": [ 630, 33 ], "matrix": "L0.mlp.W_out" } ], "l:887": [ { "axis0": [ 0, 1 ], "axis1": [ 544, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 544, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 333, 33 ], "axis1": [ 544, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 544, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 333, 33 ], "axis1": [ 544, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 544, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 544, 33 ], "axis1": [ 62, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 544, 33 ], "axis1": [ 1545, 13 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 544, 33 ], "axis1": [ 1559, 19 ], "matrix": "L4.mlp.W_out" } ], "l:889": [ { "axis0": [ 0, 1 ], "axis1": [ 333, 33 ], "matrix": "L0.mlp.W_out" } ], "l:89": [ { "axis0": [ 1022, 1 ], "axis1": [ 256, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 1804, 1 ], "axis1": [ 256, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 41, 1 ], "axis1": [ 256, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 1024, 1 ], "axis1": [ 256, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 147, 33 ], "axis1": [ 256, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 146, 6 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 158, 7 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 168, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 171, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 174, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 177, 8 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 186, 6 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 195, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 197, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 222, 1 ], "matrix": "L9.attn.W_O" } ], "l:890": [ { "axis0": [ 0, 1 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 869, 8 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 878, 1 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 881, 7 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 929, 4 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 966, 13 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1357, 1 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 869, 8 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 878, 1 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 881, 7 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 929, 4 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 966, 13 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1357, 1 ], "axis1": [ 242, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 242, 33 ], "axis1": [ 202, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 242, 33 ], "axis1": [ 204, 8 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 242, 33 ], "axis1": [ 213, 9 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 242, 33 ], "axis1": [ 235, 2 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 242, 33 ], "axis1": [ 1021, 2 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 242, 33 ], "axis1": [ 1024, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 242, 33 ], "axis1": [ 1028, 2 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 242, 33 ], "axis1": [ 1081, 1 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 242, 33 ], "axis1": [ 1179, 5 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 242, 33 ], "axis1": [ 1190, 2 ], "matrix": "L9.mlp.W_out" } ], "l:892": [ { "axis0": [ 0, 1 ], "axis1": [ 910, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 910, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1357, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1054, 2 ], "axis1": [ 910, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 910, 2 ], "axis1": [ 1357, 1 ], "matrix": "L4.mlp.W_out" } ], "l:894": [ { "axis0": [ 0, 1 ], "axis1": [ 244, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 244, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 244, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1054, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 244, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 244, 2 ], "axis1": [ 1054, 1 ], "matrix": "L3.mlp.W_out" } ], "l:895": [ { "axis0": [ 0, 1 ], "axis1": [ 246, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 246, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 246, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1055, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 246, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 246, 2 ], "axis1": [ 1055, 1 ], "matrix": "L3.mlp.W_out" } ], "l:896": [ { "axis0": [ 0, 1 ], "axis1": [ 251, 66 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 63, 1 ], "axis1": [ 251, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 251, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 669, 33 ], "axis1": [ 251, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1578, 8 ], "axis1": [ 251, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1587, 48 ], "axis1": [ 251, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1641, 1 ], "axis1": [ 251, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1700, 1 ], "axis1": [ 251, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1808, 1 ], "axis1": [ 251, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1810, 1 ], "axis1": [ 251, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1812, 1 ], "axis1": [ 251, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1838, 2 ], "axis1": [ 251, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1842, 1 ], "axis1": [ 251, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 63, 1 ], "axis1": [ 284, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 284, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 669, 33 ], "axis1": [ 284, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1578, 8 ], "axis1": [ 284, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1587, 48 ], "axis1": [ 284, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1641, 1 ], "axis1": [ 284, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1700, 1 ], "axis1": [ 284, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1808, 1 ], "axis1": [ 284, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1810, 1 ], "axis1": [ 284, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1812, 1 ], "axis1": [ 284, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1838, 2 ], "axis1": [ 284, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1842, 1 ], "axis1": [ 284, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 251, 8 ], "axis1": [ 869, 8 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 259, 1 ], "axis1": [ 878, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 260, 7 ], "axis1": [ 881, 7 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 267, 4 ], "axis1": [ 929, 4 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 271, 13 ], "axis1": [ 966, 13 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 284, 8 ], "axis1": [ 869, 8 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 292, 1 ], "axis1": [ 878, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 293, 7 ], "axis1": [ 881, 7 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 300, 4 ], "axis1": [ 929, 4 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 304, 13 ], "axis1": [ 966, 13 ], "diagonal": true, "matrix": "L8.mlp.W_out" } ], "l:898": [ { "axis0": [ 0, 1 ], "axis1": [ 577, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 577, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1151, 33 ], "axis1": [ 577, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 577, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1151, 33 ], "axis1": [ 577, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 577, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 577, 33 ], "axis1": [ 63, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 577, 33 ], "axis1": [ 1578, 8 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 577, 33 ], "axis1": [ 1587, 24 ], "matrix": "L4.mlp.W_out" } ], "l:9": [ { "axis0": [ 0, 1 ], "axis1": [ 830, 1 ], "matrix": "L0.mlp.W_out" } ], "l:900": [ { "axis0": [ 0, 1 ], "axis1": [ 1151, 33 ], "matrix": "L3.mlp.W_out" } ], "l:901": [ { "axis0": [ 0, 1 ], "axis1": [ 610, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 610, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 663, 33 ], "axis1": [ 610, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 610, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 663, 33 ], "axis1": [ 610, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 610, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 610, 33 ], "axis1": [ 65, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 610, 33 ], "axis1": [ 1611, 24 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 610, 33 ], "axis1": [ 1641, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 610, 33 ], "axis1": [ 1700, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 610, 33 ], "axis1": [ 1808, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 610, 33 ], "axis1": [ 1810, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 610, 33 ], "axis1": [ 1812, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 610, 33 ], "axis1": [ 1838, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 610, 33 ], "axis1": [ 1842, 1 ], "matrix": "L4.mlp.W_out" } ], "l:903": [ { "axis0": [ 0, 1 ], "axis1": [ 663, 33 ], "matrix": "L0.mlp.W_out" } ], "l:904": [ { "axis0": [ 0, 1 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1561, 17 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1586, 1 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1636, 2 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1639, 2 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1644, 9 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1656, 2 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1561, 17 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1586, 1 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1636, 2 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1639, 2 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1644, 9 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1656, 2 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 337, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 337, 33 ], "axis1": [ 669, 33 ], "matrix": "L7.mlp.W_out" } ], "l:906": [ { "axis0": [ 0, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 34, 2 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 37, 4 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 42, 3 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 46, 4 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 51, 4 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 61, 2 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 66, 2 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 71, 10 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 167, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1067, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1069, 4 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1078, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1085, 26 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1112, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 34, 2 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 37, 4 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 42, 3 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 46, 4 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 51, 4 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 58, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 61, 2 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 66, 2 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 69, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 71, 10 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 167, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1067, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1069, 4 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1078, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1085, 26 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1112, 1 ], "axis1": [ 598, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 598, 66 ], "axis1": [ 1561, 17 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 598, 66 ], "axis1": [ 1586, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 598, 66 ], "axis1": [ 1636, 2 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 598, 66 ], "axis1": [ 1639, 2 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 598, 66 ], "axis1": [ 1644, 9 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 598, 66 ], "axis1": [ 1656, 2 ], "matrix": "L6.mlp.W_out" } ], "l:908": [ { "axis0": [ 0, 1 ], "axis1": [ 167, 1 ], "matrix": "L0.mlp.W_out" } ], "l:909": [ { "axis0": [ 0, 1 ], "axis1": [ 1067, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1069, 4 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1078, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1085, 26 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1112, 1 ], "matrix": "L5.mlp.W_out" } ], "l:91": [ { "axis0": [ 0, 1 ], "axis1": [ 1022, 1 ], "matrix": "L8.mlp.W_out" } ], "l:910": [ { "axis0": [ 330, 6 ], "axis1": [ 0, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 929, 6 ], "axis1": [ 0, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 0, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 34, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 37, 4 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 42, 3 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 46, 4 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 51, 4 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 58, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 61, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 66, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 69, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 71, 10 ], "matrix": "L6.attn.W_O" } ], "l:911": [ { "axis0": [ 0, 1 ], "axis1": [ 69, 22 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 22 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 330, 6 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 1076, 1 ], "axis1": [ 69, 22 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 69, 22 ], "axis1": [ 330, 6 ], "matrix": "L5.mlp.W_out" } ], "l:912": [ { "axis0": [ 0, 1 ], "axis1": [ 288, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 66, 1 ], "axis1": [ 288, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 71, 1 ], "axis1": [ 288, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 139, 13 ], "axis1": [ 288, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1656, 3 ], "axis1": [ 288, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1699, 1 ], "axis1": [ 288, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1767, 14 ], "axis1": [ 288, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 289, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 71, 1 ], "axis1": [ 289, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 139, 13 ], "axis1": [ 289, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1656, 3 ], "axis1": [ 289, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1699, 1 ], "axis1": [ 289, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1767, 14 ], "axis1": [ 289, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 288, 1 ], "axis1": [ 1076, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 289, 1 ], "axis1": [ 1076, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" } ], "l:913": [ { "axis0": [ 147, 1 ], "axis1": [ 320, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 147, 1 ], "axis1": [ 320, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 320, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 66, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 71, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 139, 13 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 1656, 3 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 1699, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 1767, 14 ], "matrix": "L3.attn.W_O" } ], "l:914": [ { "axis0": [ 0, 1 ], "axis1": [ 147, 1 ], "matrix": "L0.mlp.W_out" } ], "l:915": [ { "axis0": [ 0, 1 ], "axis1": [ 549, 12 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1646, 1 ], "axis1": [ 549, 6 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1694, 1 ], "axis1": [ 549, 6 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1718, 4 ], "axis1": [ 549, 6 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1646, 1 ], "axis1": [ 555, 6 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1694, 1 ], "axis1": [ 555, 6 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1718, 4 ], "axis1": [ 555, 6 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 549, 6 ], "axis1": [ 929, 6 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 555, 6 ], "axis1": [ 929, 6 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 929, 6 ], "matrix": "L5.mlp.W_out" } ], "l:916": [ { "axis0": [ 0, 1 ], "axis1": [ 66, 24 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 66, 24 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 66, 24 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1646, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1694, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1718, 4 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1072, 1 ], "axis1": [ 66, 24 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1640, 1 ], "axis1": [ 66, 24 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 66, 24 ], "axis1": [ 1646, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 66, 24 ], "axis1": [ 1694, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 66, 24 ], "axis1": [ 1718, 4 ], "matrix": "L4.mlp.W_out" } ], "l:918": [ { "axis0": [ 0, 1 ], "axis1": [ 280, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 836, 1 ], "axis1": [ 280, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 836, 1 ], "axis1": [ 281, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 280, 1 ], "axis1": [ 1072, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 281, 1 ], "axis1": [ 1072, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 280, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 281, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1072, 1 ], "matrix": "L3.mlp.W_out" } ], "l:919": [ { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1072, 1 ], "axis1": [ 704, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 1640, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1640, 1 ], "matrix": "L4.mlp.W_out" } ], "l:92": [ { "axis0": [ 0, 1 ], "axis1": [ 1804, 1 ], "matrix": "L7.mlp.W_out" } ], "l:920": [ { "axis0": [ 0, 1 ], "axis1": [ 183, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 183, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 312, 33 ], "axis1": [ 183, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1492, 1 ], "axis1": [ 183, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 312, 33 ], "axis1": [ 183, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1492, 1 ], "axis1": [ 183, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 183, 33 ], "axis1": [ 714, 3 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 183, 33 ], "axis1": [ 762, 30 ], "matrix": "L8.mlp.W_out" } ], "l:922": [ { "axis0": [ 0, 1 ], "axis1": [ 912, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 912, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1492, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1056, 2 ], "axis1": [ 912, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 912, 2 ], "axis1": [ 1492, 1 ], "matrix": "L4.mlp.W_out" } ], "l:924": [ { "axis0": [ 0, 1 ], "axis1": [ 248, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 248, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 248, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1056, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 248, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 248, 2 ], "axis1": [ 1056, 1 ], "matrix": "L3.mlp.W_out" } ], "l:925": [ { "axis0": [ 0, 1 ], "axis1": [ 250, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 250, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 250, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1057, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 250, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 250, 2 ], "axis1": [ 1057, 1 ], "matrix": "L3.mlp.W_out" } ], "l:926": [ { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 702, 15 ], "axis1": [ 704, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 828, 6 ], "axis1": [ 704, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 836, 4 ], "axis1": [ 704, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 841, 4 ], "axis1": [ 704, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 847, 3 ], "axis1": [ 704, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 851, 1 ], "axis1": [ 704, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1701, 1 ], "axis1": [ 704, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1845, 30 ], "axis1": [ 704, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 312, 33 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 853, 2 ], "axis1": [ 768, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 856, 4 ], "axis1": [ 768, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 862, 15 ], "axis1": [ 768, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 878, 1 ], "axis1": [ 768, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 881, 7 ], "axis1": [ 768, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 929, 4 ], "axis1": [ 768, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1875, 2 ], "axis1": [ 768, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 312, 33 ], "matrix": "L8.attn.W_O" } ], "l:928": [ { "axis0": [ 0, 1 ], "axis1": [ 370, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 370, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 432, 33 ], "axis1": [ 370, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 370, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 432, 33 ], "axis1": [ 370, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 370, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 370, 33 ], "axis1": [ 702, 15 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 370, 33 ], "axis1": [ 828, 6 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 370, 33 ], "axis1": [ 836, 4 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 370, 33 ], "axis1": [ 841, 4 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 370, 33 ], "axis1": [ 847, 3 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 370, 33 ], "axis1": [ 851, 1 ], "matrix": "L7.mlp.W_out" } ], "l:930": [ { "axis0": [ 0, 1 ], "axis1": [ 432, 33 ], "matrix": "L0.mlp.W_out" } ], "l:931": [ { "axis0": [ 0, 1 ], "axis1": [ 643, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 643, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 696, 33 ], "axis1": [ 643, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 643, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 696, 33 ], "axis1": [ 643, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 643, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 643, 33 ], "axis1": [ 1701, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 643, 33 ], "axis1": [ 1845, 32 ], "matrix": "L4.mlp.W_out" } ], "l:933": [ { "axis0": [ 0, 1 ], "axis1": [ 696, 33 ], "matrix": "L0.mlp.W_out" } ], "l:934": [ { "axis0": [ 0, 1 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1658, 19 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1679, 2 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1683, 7 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1694, 4 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1699, 1 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1702, 1 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1658, 19 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1679, 2 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1683, 7 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1694, 4 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1699, 1 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1702, 1 ], "axis1": [ 403, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 403, 33 ], "axis1": [ 853, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 403, 33 ], "axis1": [ 856, 4 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 403, 33 ], "axis1": [ 862, 15 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 403, 33 ], "axis1": [ 878, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 403, 33 ], "axis1": [ 881, 7 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 403, 33 ], "axis1": [ 929, 4 ], "matrix": "L7.mlp.W_out" } ], "l:936": [ { "axis0": [ 0, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 83, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 120, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 135, 31 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1067, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1069, 4 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1078, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1085, 26 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1112, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1725, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 83, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 120, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 135, 31 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1067, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1069, 4 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1078, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1085, 26 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1112, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1725, 1 ], "axis1": [ 664, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 664, 66 ], "axis1": [ 1658, 19 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 664, 66 ], "axis1": [ 1680, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 664, 66 ], "axis1": [ 1683, 7 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 664, 66 ], "axis1": [ 1694, 4 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 664, 66 ], "axis1": [ 1699, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 664, 66 ], "axis1": [ 1702, 1 ], "matrix": "L6.mlp.W_out" } ], "l:938": [ { "axis0": [ 0, 1 ], "axis1": [ 110, 6 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 110, 6 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1725, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1077, 1 ], "axis1": [ 110, 6 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 110, 6 ], "axis1": [ 1725, 1 ], "matrix": "L4.mlp.W_out" } ], "l:939": [ { "axis0": [ 0, 1 ], "axis1": [ 290, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 152, 9 ], "axis1": [ 290, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 828, 8 ], "axis1": [ 290, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1659, 3 ], "axis1": [ 290, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1781, 9 ], "axis1": [ 290, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1791, 4 ], "axis1": [ 290, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 152, 9 ], "axis1": [ 291, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 828, 8 ], "axis1": [ 291, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1659, 3 ], "axis1": [ 291, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1781, 9 ], "axis1": [ 291, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1791, 4 ], "axis1": [ 291, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 290, 1 ], "axis1": [ 1077, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 291, 1 ], "axis1": [ 1077, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" } ], "l:94": [ { "axis0": [ 0, 1 ], "axis1": [ 1024, 1 ], "matrix": "L8.mlp.W_out" } ], "l:940": [ { "axis0": [ 1841, 1 ], "axis1": [ 384, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 1841, 1 ], "axis1": [ 384, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 384, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 152, 9 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 828, 8 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 1659, 3 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 1781, 9 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 1791, 4 ], "matrix": "L3.attn.W_O" } ], "l:941": [ { "axis0": [ 0, 1 ], "axis1": [ 1841, 1 ], "matrix": "L1.mlp.W_out" } ], "l:942": [ { "axis0": [ 324, 6 ], "axis1": [ 64, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 929, 6 ], "axis1": [ 64, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 64, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 83, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 120, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 135, 31 ], "matrix": "L6.attn.W_O" } ], "l:943": [ { "axis0": [ 0, 1 ], "axis1": [ 47, 22 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 47, 22 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 324, 6 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 1077, 1 ], "axis1": [ 47, 22 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 47, 22 ], "axis1": [ 324, 6 ], "matrix": "L5.mlp.W_out" } ], "l:944": [ { "axis0": [ 0, 1 ], "axis1": [ 1801, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1801, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 480, 33 ], "axis1": [ 1801, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1111, 1 ], "axis1": [ 1801, 33 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 480, 33 ], "axis1": [ 1801, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 1111, 1 ], "axis1": [ 1801, 33 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 1801, 33 ], "axis1": [ 140, 3 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 1801, 33 ], "axis1": [ 152, 6 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 1801, 33 ], "axis1": [ 165, 3 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 1801, 33 ], "axis1": [ 169, 1 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 1801, 33 ], "axis1": [ 193, 2 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 1801, 33 ], "axis1": [ 684, 18 ], "matrix": "L11.mlp.W_out" } ], "l:946": [ { "axis0": [ 0, 1 ], "axis1": [ 914, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 914, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1111, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1058, 2 ], "axis1": [ 914, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 914, 2 ], "axis1": [ 1111, 1 ], "matrix": "L4.mlp.W_out" } ], "l:948": [ { "axis0": [ 0, 1 ], "axis1": [ 252, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 252, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 252, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1058, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 252, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 252, 2 ], "axis1": [ 1058, 1 ], "matrix": "L3.mlp.W_out" } ], "l:949": [ { "axis0": [ 0, 1 ], "axis1": [ 254, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 254, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 254, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1059, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 254, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 254, 2 ], "axis1": [ 1059, 1 ], "matrix": "L3.mlp.W_out" } ], "l:95": [ { "axis0": [ 81, 1 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 1709, 3 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 1800, 4 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 1190, 33 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 147, 33 ], "matrix": "L8.attn.W_O" } ], "l:950": [ { "axis0": [ 0, 1 ], "axis1": [ 36, 66 ], "matrix": "L10.mlp.W_up" }, { "axis0": [ 130, 3 ], "axis1": [ 36, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 669, 30 ], "axis1": [ 36, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 933, 33 ], "axis1": [ 36, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 1877, 33 ], "axis1": [ 36, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 130, 3 ], "axis1": [ 69, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 669, 30 ], "axis1": [ 69, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 933, 33 ], "axis1": [ 69, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 1877, 33 ], "axis1": [ 69, 33 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 36, 33 ], "axis1": [ 480, 33 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 69, 33 ], "axis1": [ 480, 33 ], "diagonal": true, "matrix": "L10.mlp.W_out" } ], "l:952": [ { "axis0": [ 0, 1 ], "axis1": [ 676, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 676, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 465, 33 ], "axis1": [ 676, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 676, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 465, 33 ], "axis1": [ 676, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 676, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 676, 33 ], "axis1": [ 1877, 33 ], "matrix": "L4.mlp.W_out" } ], "l:954": [ { "axis0": [ 0, 1 ], "axis1": [ 465, 33 ], "matrix": "L0.mlp.W_out" } ], "l:955": [ { "axis0": [ 0, 1 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1030, 11 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1042, 4 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1047, 4 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1052, 9 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1062, 1 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1064, 4 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1030, 11 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1042, 4 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1047, 4 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1052, 9 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1062, 1 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1064, 4 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 176, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 176, 33 ], "axis1": [ 130, 3 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 176, 33 ], "axis1": [ 669, 30 ], "matrix": "L9.mlp.W_out" } ], "l:957": [ { "axis0": [ 0, 1 ], "axis1": [ 1030, 11 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1042, 4 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1047, 4 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1052, 9 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1062, 1 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1064, 4 ], "matrix": "L8.mlp.W_out" } ], "l:958": [ { "axis0": [ 0, 1 ], "axis1": [ 436, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 436, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 436, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1705, 1 ], "axis1": [ 436, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1707, 2 ], "axis1": [ 436, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1712, 12 ], "axis1": [ 436, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1726, 18 ], "axis1": [ 436, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 436, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1705, 1 ], "axis1": [ 436, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1707, 2 ], "axis1": [ 436, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1712, 12 ], "axis1": [ 436, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1726, 18 ], "axis1": [ 436, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 436, 33 ], "axis1": [ 933, 33 ], "matrix": "L7.mlp.W_out" } ], "l:96": [ { "axis0": [ 0, 1 ], "axis1": [ 18, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 18, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 81, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1709, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1677, 1 ], "axis1": [ 18, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 18, 17 ], "axis1": [ 81, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 1709, 3 ], "matrix": "L4.mlp.W_out" } ], "l:960": [ { "axis0": [ 0, 1 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 166, 1 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 168, 32 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1067, 1 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1069, 4 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1078, 1 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1085, 26 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1112, 1 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1635, 1 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 166, 1 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 168, 32 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1067, 1 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1069, 4 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1078, 1 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1085, 26 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1112, 1 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1635, 1 ], "axis1": [ 730, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 730, 66 ], "axis1": [ 1705, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 730, 66 ], "axis1": [ 1707, 2 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 730, 66 ], "axis1": [ 1712, 12 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 730, 66 ], "axis1": [ 1726, 18 ], "matrix": "L6.mlp.W_out" } ], "l:962": [ { "axis0": [ 0, 1 ], "axis1": [ 116, 6 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 116, 6 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1635, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 1013, 1 ], "axis1": [ 116, 6 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 116, 6 ], "axis1": [ 1635, 1 ], "matrix": "L3.mlp.W_out" } ], "l:963": [ { "axis0": [ 0, 1 ], "axis1": [ 147, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 66, 1 ], "axis1": [ 147, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 70, 1 ], "axis1": [ 147, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 125, 12 ], "axis1": [ 147, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1654, 3 ], "axis1": [ 147, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1753, 16 ], "axis1": [ 147, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 148, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 70, 1 ], "axis1": [ 148, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 125, 12 ], "axis1": [ 148, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1654, 3 ], "axis1": [ 148, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1753, 16 ], "axis1": [ 148, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1013, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 148, 1 ], "axis1": [ 1013, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" } ], "l:964": [ { "axis0": [ 1842, 1 ], "axis1": [ 384, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 1842, 1 ], "axis1": [ 384, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 384, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 66, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 70, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 125, 12 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 1654, 3 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 1753, 16 ], "matrix": "L2.attn.W_O" } ], "l:965": [ { "axis0": [ 0, 1 ], "axis1": [ 1842, 1 ], "matrix": "L1.mlp.W_out" } ], "l:966": [ { "axis0": [ 865, 6 ], "axis1": [ 128, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 929, 6 ], "axis1": [ 128, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 128, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 166, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 168, 32 ], "matrix": "L6.attn.W_O" } ], "l:967": [ { "axis0": [ 0, 1 ], "axis1": [ 34, 17 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 34, 17 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 865, 6 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 1013, 1 ], "axis1": [ 34, 17 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 34, 17 ], "axis1": [ 865, 6 ], "matrix": "L3.mlp.W_out" } ], "l:968": [ { "axis0": [ 0, 1 ], "axis1": [ 216, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 216, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 345, 8 ], "axis1": [ 216, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 419, 25 ], "axis1": [ 216, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1369, 1 ], "axis1": [ 216, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 345, 8 ], "axis1": [ 216, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 419, 25 ], "axis1": [ 216, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1369, 1 ], "axis1": [ 216, 33 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 216, 33 ], "axis1": [ 792, 3 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 216, 33 ], "axis1": [ 828, 6 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 216, 33 ], "axis1": [ 836, 4 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 216, 33 ], "axis1": [ 841, 4 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 216, 33 ], "axis1": [ 847, 3 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 216, 33 ], "axis1": [ 851, 1 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 216, 33 ], "axis1": [ 853, 2 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 216, 33 ], "axis1": [ 856, 4 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 216, 33 ], "axis1": [ 862, 6 ], "matrix": "L8.mlp.W_out" } ], "l:97": [ { "axis0": [ 1080, 1 ], "axis1": [ 256, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 1843, 1 ], "axis1": [ 256, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 81, 1 ], "axis1": [ 256, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1796, 1 ], "axis1": [ 256, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1068, 1 ], "axis1": [ 256, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 1677, 1 ], "matrix": "L4.attn.W_O" } ], "l:970": [ { "axis0": [ 0, 1 ], "axis1": [ 916, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 916, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1369, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 42, 1 ], "axis1": [ 916, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1060, 1 ], "axis1": [ 916, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 916, 2 ], "axis1": [ 1369, 1 ], "matrix": "L4.mlp.W_out" } ], "l:972": [ { "axis0": [ 0, 1 ], "axis1": [ 256, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1060, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 256, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 256, 2 ], "axis1": [ 1060, 1 ], "matrix": "L3.mlp.W_out" } ], "l:973": [ { "axis0": [ 0, 1 ], "axis1": [ 258, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 258, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 258, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 42, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 258, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 258, 2 ], "axis1": [ 42, 1 ], "matrix": "L3.mlp.W_out" } ], "l:974": [ { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 1910, 64 ], "axis1": [ 832, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 345, 8 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 419, 25 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 966, 16 ], "axis1": [ 896, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 983, 5 ], "axis1": [ 896, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 993, 12 ], "axis1": [ 896, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1974, 2 ], "axis1": [ 896, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 345, 8 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 419, 25 ], "matrix": "L8.attn.W_O" } ], "l:976": [ { "axis0": [ 0, 1 ], "axis1": [ 709, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 709, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 498, 33 ], "axis1": [ 709, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1678, 1 ], "axis1": [ 709, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 498, 33 ], "axis1": [ 709, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1678, 1 ], "axis1": [ 709, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 709, 33 ], "axis1": [ 1910, 33 ], "matrix": "L4.mlp.W_out" } ], "l:978": [ { "axis0": [ 0, 1 ], "axis1": [ 498, 33 ], "matrix": "L0.mlp.W_out" } ], "l:979": [ { "axis0": [ 0, 1 ], "axis1": [ 742, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 742, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 729, 33 ], "axis1": [ 742, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 742, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 729, 33 ], "axis1": [ 742, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 742, 33 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 742, 33 ], "axis1": [ 1943, 33 ], "matrix": "L4.mlp.W_out" } ], "l:981": [ { "axis0": [ 0, 1 ], "axis1": [ 729, 33 ], "matrix": "L0.mlp.W_out" } ], "l:982": [ { "axis0": [ 0, 1 ], "axis1": [ 469, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 469, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 469, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1744, 6 ], "axis1": [ 469, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1763, 27 ], "axis1": [ 469, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 469, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1744, 6 ], "axis1": [ 469, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1763, 27 ], "axis1": [ 469, 33 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 469, 33 ], "axis1": [ 966, 16 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 469, 33 ], "axis1": [ 983, 5 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 469, 33 ], "axis1": [ 993, 12 ], "matrix": "L7.mlp.W_out" } ], "l:984": [ { "axis0": [ 0, 1 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 200, 33 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1067, 1 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1069, 4 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1078, 1 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1085, 26 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1112, 1 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1804, 1 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 200, 33 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1067, 1 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1069, 4 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1078, 1 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1085, 26 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1112, 1 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1804, 1 ], "axis1": [ 796, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 796, 66 ], "axis1": [ 1744, 6 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 796, 66 ], "axis1": [ 1763, 27 ], "matrix": "L6.mlp.W_out" } ], "l:986": [ { "axis0": [ 0, 1 ], "axis1": [ 110, 6 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 110, 6 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1804, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 73, 1 ], "axis1": [ 110, 6 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 110, 6 ], "axis1": [ 1804, 1 ], "matrix": "L3.mlp.W_out" } ], "l:987": [ { "axis0": [ 0, 1 ], "axis1": [ 149, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 71, 1 ], "axis1": [ 149, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 140, 5 ], "axis1": [ 149, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 148, 8 ], "axis1": [ 149, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1657, 3 ], "axis1": [ 149, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1699, 1 ], "axis1": [ 149, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1769, 15 ], "axis1": [ 149, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 71, 1 ], "axis1": [ 150, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 140, 5 ], "axis1": [ 150, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 148, 8 ], "axis1": [ 150, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1657, 3 ], "axis1": [ 150, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1699, 1 ], "axis1": [ 150, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1769, 15 ], "axis1": [ 150, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 73, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 150, 1 ], "axis1": [ 73, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" } ], "l:988": [ { "axis0": [ 148, 1 ], "axis1": [ 448, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 148, 1 ], "axis1": [ 448, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 448, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 71, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 140, 5 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 148, 8 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1657, 3 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1699, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1769, 15 ], "matrix": "L2.attn.W_O" } ], "l:989": [ { "axis0": [ 0, 1 ], "axis1": [ 148, 1 ], "matrix": "L0.mlp.W_out" } ], "l:99": [ { "axis0": [ 0, 1 ], "axis1": [ 1080, 1 ], "matrix": "L3.mlp.W_out" } ], "l:990": [ { "axis0": [ 34, 1 ], "axis1": [ 192, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 859, 1 ], "axis1": [ 192, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 862, 3 ], "axis1": [ 192, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 1799, 1 ], "axis1": [ 192, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 929, 6 ], "axis1": [ 192, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1, 33 ], "axis1": [ 192, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 200, 33 ], "matrix": "L6.attn.W_O" } ], "l:991": [ { "axis0": [ 0, 1 ], "axis1": [ 21, 13 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 21, 13 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 34, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 859, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 862, 3 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1799, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 73, 1 ], "axis1": [ 21, 13 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 21, 13 ], "axis1": [ 34, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 21, 13 ], "axis1": [ 859, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 21, 13 ], "axis1": [ 862, 3 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 21, 13 ], "axis1": [ 1799, 1 ], "matrix": "L3.mlp.W_out" } ], "l:992": [ { "axis0": [ 0, 1 ], "axis1": [ 275, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 275, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 444, 33 ], "axis1": [ 275, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 1382, 1 ], "axis1": [ 275, 33 ], "matrix": "L9.mlp.W_in" }, { "axis0": [ 444, 33 ], "axis1": [ 275, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 1382, 1 ], "axis1": [ 275, 33 ], "matrix": "L9.mlp.W_up" }, { "axis0": [ 275, 33 ], "axis1": [ 242, 2 ], "matrix": "L9.mlp.W_out" }, { "axis0": [ 275, 33 ], "axis1": [ 1192, 31 ], "matrix": "L9.mlp.W_out" } ], "l:994": [ { "axis0": [ 0, 1 ], "axis1": [ 918, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 918, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1382, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 1062, 1 ], "axis1": [ 918, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1066, 1 ], "axis1": [ 918, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 918, 2 ], "axis1": [ 1382, 1 ], "matrix": "L4.mlp.W_out" } ], "l:996": [ { "axis0": [ 0, 1 ], "axis1": [ 260, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 260, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 260, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1062, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 260, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 260, 2 ], "axis1": [ 1062, 1 ], "matrix": "L3.mlp.W_out" } ], "l:997": [ { "axis0": [ 0, 1 ], "axis1": [ 262, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 262, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 262, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1066, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 836, 1 ], "axis1": [ 262, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 262, 2 ], "axis1": [ 1066, 1 ], "matrix": "L3.mlp.W_out" } ], "l:998": [ { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 1005, 15 ], "axis1": [ 960, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1021, 2 ], "axis1": [ 960, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1024, 17 ], "axis1": [ 960, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1042, 4 ], "axis1": [ 960, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1047, 4 ], "axis1": [ 960, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1052, 9 ], "axis1": [ 960, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1062, 1 ], "axis1": [ 960, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1064, 12 ], "axis1": [ 960, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 960, 64 ], "axis1": [ 444, 33 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 1076, 5 ], "axis1": [ 1024, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1082, 29 ], "axis1": [ 1024, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1112, 1 ], "axis1": [ 1024, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1024, 64 ], "axis1": [ 444, 33 ], "matrix": "L8.attn.W_O" } ], "physical:cancel": [ { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 44, 1 ], "axis1": [ 1024, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 58, 1 ], "axis1": [ 1024, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 66, 1 ], "axis1": [ 1024, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 73, 4 ], "axis1": [ 1024, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 83, 1 ], "axis1": [ 1024, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 124, 1 ], "axis1": [ 1024, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 130, 1 ], "axis1": [ 1024, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 148, 3 ], "axis1": [ 1024, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 160, 1 ], "axis1": [ 1024, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 831, 50 ], "axis1": [ 1024, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1024, 64 ], "axis1": [ 44, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 58, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 66, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 73, 4 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 83, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 124, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 130, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 148, 3 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 160, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 831, 50 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 881, 64 ], "axis1": [ 1088, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1088, 64 ], "axis1": [ 881, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 945, 64 ], "axis1": [ 1152, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1152, 64 ], "axis1": [ 945, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1009, 64 ], "axis1": [ 1216, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1216, 64 ], "axis1": [ 1009, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1073, 64 ], "axis1": [ 1280, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1280, 64 ], "axis1": [ 1073, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1137, 64 ], "axis1": [ 1344, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1344, 64 ], "axis1": [ 1137, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1201, 64 ], "axis1": [ 1408, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1408, 64 ], "axis1": [ 1201, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1265, 64 ], "axis1": [ 1472, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1472, 64 ], "axis1": [ 1265, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1329, 64 ], "axis1": [ 1536, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1536, 64 ], "axis1": [ 1329, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1393, 64 ], "axis1": [ 1600, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1600, 64 ], "axis1": [ 1393, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1457, 64 ], "axis1": [ 1664, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1664, 64 ], "axis1": [ 1457, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1728, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1728, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1521, 64 ], "axis1": [ 1728, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1728, 64 ], "axis1": [ 1521, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1792, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1792, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1585, 64 ], "axis1": [ 1792, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1792, 64 ], "axis1": [ 1585, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1856, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1856, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1649, 64 ], "axis1": [ 1856, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1856, 64 ], "axis1": [ 1649, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1920, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1920, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1713, 64 ], "axis1": [ 1920, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1920, 64 ], "axis1": [ 1713, 64 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1984, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1984, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 1777, 13 ], "axis1": [ 1984, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1823, 1 ], "axis1": [ 1984, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1831, 2 ], "axis1": [ 1984, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1835, 3 ], "axis1": [ 1984, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1840, 1 ], "axis1": [ 1984, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1842, 3 ], "axis1": [ 1984, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 1984, 64 ], "axis1": [ 1777, 13 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1984, 64 ], "axis1": [ 1823, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1984, 64 ], "axis1": [ 1831, 2 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1984, 64 ], "axis1": [ 1835, 3 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1984, 64 ], "axis1": [ 1840, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 1984, 64 ], "axis1": [ 1842, 3 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 34, 1 ], "axis1": [ 1024, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 46, 1 ], "axis1": [ 1024, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 66, 1 ], "axis1": [ 1024, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 70, 3 ], "axis1": [ 1024, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 83, 1 ], "axis1": [ 1024, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 120, 17 ], "axis1": [ 1024, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 138, 1 ], "axis1": [ 1024, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 140, 21 ], "axis1": [ 1024, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 828, 18 ], "axis1": [ 1024, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1024, 64 ], "axis1": [ 34, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 46, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 66, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 70, 3 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 83, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 120, 17 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 138, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 140, 21 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 828, 18 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 846, 15 ], "axis1": [ 1088, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 862, 27 ], "axis1": [ 1088, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 890, 22 ], "axis1": [ 1088, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1088, 64 ], "axis1": [ 846, 15 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1088, 64 ], "axis1": [ 862, 27 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1088, 64 ], "axis1": [ 890, 22 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 912, 64 ], "axis1": [ 1152, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1152, 64 ], "axis1": [ 912, 64 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 976, 4 ], "axis1": [ 1216, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 981, 1 ], "axis1": [ 1216, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 987, 1 ], "axis1": [ 1216, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 999, 4 ], "axis1": [ 1216, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1004, 4 ], "axis1": [ 1216, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1635, 1 ], "axis1": [ 1216, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1642, 1 ], "axis1": [ 1216, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1644, 20 ], "axis1": [ 1216, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1694, 1 ], "axis1": [ 1216, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1697, 3 ], "axis1": [ 1216, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1707, 24 ], "axis1": [ 1216, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1216, 64 ], "axis1": [ 976, 4 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 981, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 987, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 999, 4 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1004, 4 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1635, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1642, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1644, 20 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1694, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1697, 3 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1707, 24 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1731, 59 ], "axis1": [ 1280, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1791, 2 ], "axis1": [ 1280, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1794, 3 ], "axis1": [ 1280, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1280, 64 ], "axis1": [ 1731, 59 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 1791, 2 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 1794, 3 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 1797, 1 ], "axis1": [ 1344, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1804, 1 ], "axis1": [ 1344, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1806, 1 ], "axis1": [ 1344, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1808, 1 ], "axis1": [ 1344, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1810, 1 ], "axis1": [ 1344, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1812, 1 ], "axis1": [ 1344, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1817, 3 ], "axis1": [ 1344, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1823, 1 ], "axis1": [ 1344, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1838, 2 ], "axis1": [ 1344, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1841, 1 ], "axis1": [ 1344, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 1344, 64 ], "axis1": [ 1797, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1804, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1806, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1808, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1810, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1812, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1817, 3 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1823, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1838, 2 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1841, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 38, 3 ], "axis1": [ 768, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 46, 3 ], "axis1": [ 768, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 64, 8 ], "axis1": [ 768, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 73, 1 ], "axis1": [ 768, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 81, 1 ], "axis1": [ 768, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 83, 1 ], "axis1": [ 768, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 120, 17 ], "axis1": [ 768, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 138, 29 ], "axis1": [ 768, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 828, 1 ], "axis1": [ 768, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 38, 3 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 46, 3 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 64, 8 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 73, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 81, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 83, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 120, 17 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 138, 29 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 828, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 829, 11 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 841, 4 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 985, 1 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 994, 4 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1003, 1 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1009, 5 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1015, 3 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1068, 1 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1079, 2 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1082, 3 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1636, 3 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1640, 1 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1642, 20 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1664, 5 ], "axis1": [ 832, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 829, 11 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 841, 4 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 985, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 994, 4 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1003, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1009, 5 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1015, 3 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1068, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1079, 2 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1082, 3 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1636, 3 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1640, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1642, 20 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1664, 5 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1669, 12 ], "axis1": [ 896, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1688, 12 ], "axis1": [ 896, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1702, 2 ], "axis1": [ 896, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1706, 38 ], "axis1": [ 896, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 1669, 12 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 1688, 12 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 1702, 2 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 1706, 38 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 1744, 46 ], "axis1": [ 960, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1791, 5 ], "axis1": [ 960, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 1843, 1 ], "axis1": [ 960, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 960, 64 ], "axis1": [ 1744, 46 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 960, 64 ], "axis1": [ 1791, 5 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 960, 64 ], "axis1": [ 1843, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 35, 1 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 48, 1 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 51, 2 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 54, 1 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 58, 1 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 61, 2 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 67, 1 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 72, 1 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 75, 2 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 137, 1 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 267, 33 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 333, 18 ], "axis1": [ 768, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 35, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 48, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 51, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 54, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 58, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 61, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 67, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 72, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 75, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 137, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 267, 33 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 333, 18 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 351, 15 ], "axis1": [ 832, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 399, 33 ], "axis1": [ 832, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 465, 16 ], "axis1": [ 832, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 351, 15 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 399, 33 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 465, 16 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 481, 50 ], "axis1": [ 896, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 597, 14 ], "axis1": [ 896, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 481, 50 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 597, 14 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 611, 64 ], "axis1": [ 960, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 960, 64 ], "axis1": [ 611, 64 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 675, 64 ], "axis1": [ 1024, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1024, 64 ], "axis1": [ 675, 64 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 739, 23 ], "axis1": [ 1088, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 795, 33 ], "axis1": [ 1088, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 834, 2 ], "axis1": [ 1088, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 840, 1 ], "axis1": [ 1088, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 845, 2 ], "axis1": [ 1088, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 850, 1 ], "axis1": [ 1088, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 852, 1 ], "axis1": [ 1088, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 855, 1 ], "axis1": [ 1088, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1088, 64 ], "axis1": [ 739, 23 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1088, 64 ], "axis1": [ 795, 33 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1088, 64 ], "axis1": [ 834, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1088, 64 ], "axis1": [ 840, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1088, 64 ], "axis1": [ 845, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1088, 64 ], "axis1": [ 850, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1088, 64 ], "axis1": [ 852, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1088, 64 ], "axis1": [ 855, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 860, 2 ], "axis1": [ 1152, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 877, 1 ], "axis1": [ 1152, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 879, 2 ], "axis1": [ 1152, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 888, 33 ], "axis1": [ 1152, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 930, 1 ], "axis1": [ 1152, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 935, 1 ], "axis1": [ 1152, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 940, 1 ], "axis1": [ 1152, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 945, 1 ], "axis1": [ 1152, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 950, 1 ], "axis1": [ 1152, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 954, 21 ], "axis1": [ 1152, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1152, 64 ], "axis1": [ 860, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 877, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 879, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 888, 33 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 930, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 935, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 940, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 945, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 950, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 954, 21 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 975, 7 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 986, 2 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 993, 1 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 999, 4 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1004, 1 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1014, 1 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1022, 1 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1024, 1 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1026, 1 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1028, 1 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1033, 1 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1038, 3 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1044, 2 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1047, 2 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1054, 2 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1058, 2 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1065, 3 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1069, 3 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1078, 1 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1151, 25 ], "axis1": [ 1216, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1216, 64 ], "axis1": [ 975, 7 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 986, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 993, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 999, 4 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1004, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1014, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1022, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1024, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1026, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1028, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1033, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1038, 3 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1044, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1047, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1054, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1058, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1065, 3 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1069, 3 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1078, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1151, 25 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 1176, 8 ], "axis1": [ 1280, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1215, 56 ], "axis1": [ 1280, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1280, 64 ], "axis1": [ 1176, 8 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 1215, 56 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 1271, 43 ], "axis1": [ 1344, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1380, 2 ], "axis1": [ 1344, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1383, 3 ], "axis1": [ 1344, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1387, 2 ], "axis1": [ 1344, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1390, 14 ], "axis1": [ 1344, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1344, 64 ], "axis1": [ 1271, 43 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1380, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1383, 3 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1387, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1390, 14 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 1404, 28 ], "axis1": [ 1408, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1433, 24 ], "axis1": [ 1408, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1458, 3 ], "axis1": [ 1408, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1462, 9 ], "axis1": [ 1408, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1408, 64 ], "axis1": [ 1404, 28 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1408, 64 ], "axis1": [ 1433, 24 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1408, 64 ], "axis1": [ 1458, 3 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1408, 64 ], "axis1": [ 1462, 9 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 1471, 11 ], "axis1": [ 1472, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1483, 9 ], "axis1": [ 1472, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1493, 42 ], "axis1": [ 1472, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1536, 2 ], "axis1": [ 1472, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1472, 64 ], "axis1": [ 1471, 11 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 1483, 9 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 1493, 42 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 1536, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 1538, 20 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1559, 19 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1639, 2 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1645, 1 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1676, 1 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1683, 6 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1705, 1 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1712, 6 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1805, 3 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1809, 1 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1811, 1 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1818, 3 ], "axis1": [ 1536, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1536, 64 ], "axis1": [ 1538, 20 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1559, 19 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1639, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1645, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1676, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1683, 6 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1705, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1712, 6 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1805, 3 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1809, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1811, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1818, 3 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 1821, 1 ], "axis1": [ 1600, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1823, 1 ], "axis1": [ 1600, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1835, 2 ], "axis1": [ 1600, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1840, 1 ], "axis1": [ 1600, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1976, 59 ], "axis1": [ 1600, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1600, 64 ], "axis1": [ 1821, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1823, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1835, 2 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1840, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1976, 59 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 2035, 11 ], "axis1": [ 1664, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 1664, 64 ], "axis1": [ 2035, 11 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 34, 2 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 37, 1 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 40, 1 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 42, 1 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 44, 1 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 46, 4 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 51, 4 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 58, 1 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 61, 2 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 67, 1 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 69, 1 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 71, 10 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 83, 1 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 120, 1 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 137, 1 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 150, 14 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 168, 18 ], "axis1": [ 1152, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1152, 64 ], "axis1": [ 34, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 37, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 40, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 42, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 44, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 46, 4 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 51, 4 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 58, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 61, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 67, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 69, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 71, 10 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 83, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 120, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 137, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 150, 14 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 168, 18 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 186, 64 ], "axis1": [ 1216, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1216, 64 ], "axis1": [ 186, 64 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 250, 64 ], "axis1": [ 1280, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1280, 64 ], "axis1": [ 250, 64 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 314, 9 ], "axis1": [ 1344, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 324, 27 ], "axis1": [ 1344, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 617, 28 ], "axis1": [ 1344, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1344, 64 ], "axis1": [ 314, 9 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 324, 27 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 617, 28 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 645, 64 ], "axis1": [ 1408, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1408, 64 ], "axis1": [ 645, 64 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 709, 7 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 831, 1 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 847, 3 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 851, 1 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 853, 2 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 856, 4 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 862, 15 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 878, 1 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 885, 2 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 929, 6 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 998, 1 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1003, 1 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1009, 5 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1015, 3 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1021, 1 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1050, 1 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1068, 1 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1073, 5 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1079, 2 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1082, 2 ], "axis1": [ 1472, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1472, 64 ], "axis1": [ 709, 7 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 831, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 847, 3 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 851, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 853, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 856, 4 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 862, 15 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 878, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 885, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 929, 6 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 998, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 1003, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 1009, 5 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 1015, 3 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 1021, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 1050, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 1068, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 1073, 5 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 1079, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 1082, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1084, 1 ], "axis1": [ 1536, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1118, 33 ], "axis1": [ 1536, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1184, 30 ], "axis1": [ 1536, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1536, 64 ], "axis1": [ 1084, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1118, 33 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1184, 30 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1214, 1 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1389, 1 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1586, 1 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1636, 2 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1644, 1 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1646, 7 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1658, 6 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1668, 8 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1680, 1 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1694, 4 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1702, 1 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1707, 2 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1718, 6 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1726, 23 ], "axis1": [ 1600, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1600, 64 ], "axis1": [ 1214, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1389, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1586, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1636, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1644, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1646, 7 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1658, 6 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1668, 8 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1680, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1694, 4 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1702, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1707, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1718, 6 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1726, 23 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 1749, 1 ], "axis1": [ 1664, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1778, 12 ], "axis1": [ 1664, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1791, 1 ], "axis1": [ 1664, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1797, 3 ], "axis1": [ 1664, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1837, 1 ], "axis1": [ 1664, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1843, 1 ], "axis1": [ 1664, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 1664, 64 ], "axis1": [ 1749, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 1778, 12 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 1791, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 1797, 3 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 1837, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 1843, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 34, 2 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 37, 4 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 42, 3 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 46, 4 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 51, 4 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 58, 1 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 61, 2 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 66, 2 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 69, 1 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 71, 10 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 83, 1 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 85, 30 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 34, 2 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 37, 4 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 42, 3 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 46, 4 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 51, 4 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 58, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 61, 2 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 66, 2 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 69, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 71, 10 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 83, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 85, 30 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 115, 3 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 120, 1 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 135, 32 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 168, 28 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 115, 3 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 120, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 135, 32 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 168, 28 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 196, 64 ], "axis1": [ 256, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 196, 64 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 260, 63 ], "axis1": [ 320, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 324, 1 ], "axis1": [ 320, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 260, 63 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 324, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 325, 26 ], "axis1": [ 384, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 617, 38 ], "axis1": [ 384, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 325, 26 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 617, 38 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 655, 62 ], "axis1": [ 448, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 828, 2 ], "axis1": [ 448, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 655, 62 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 828, 2 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 830, 4 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 836, 4 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 841, 4 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 847, 3 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 851, 1 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 853, 2 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 856, 4 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 862, 15 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 878, 1 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 882, 2 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 885, 3 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 929, 6 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 968, 14 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 983, 1 ], "axis1": [ 512, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 830, 4 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 836, 4 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 841, 4 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 847, 3 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 851, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 853, 2 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 856, 4 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 862, 15 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 878, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 882, 2 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 885, 3 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 929, 6 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 968, 14 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 983, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 984, 4 ], "axis1": [ 576, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 993, 15 ], "axis1": [ 576, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1009, 11 ], "axis1": [ 576, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1021, 2 ], "axis1": [ 576, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1024, 4 ], "axis1": [ 576, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1050, 1 ], "axis1": [ 576, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1068, 1 ], "axis1": [ 576, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1073, 5 ], "axis1": [ 576, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1079, 2 ], "axis1": [ 576, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1082, 3 ], "axis1": [ 576, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1113, 16 ], "axis1": [ 576, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 984, 4 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 993, 15 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 1009, 11 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 1021, 2 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 1024, 4 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 1050, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 1068, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 1073, 5 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 1079, 2 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 1082, 3 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 1113, 16 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 1129, 54 ], "axis1": [ 640, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1184, 3 ], "axis1": [ 640, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1635, 1 ], "axis1": [ 640, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1724, 1 ], "axis1": [ 640, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 1804, 1 ], "axis1": [ 640, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 1129, 54 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 1184, 3 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 1635, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 1724, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 1804, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 34, 2 ], "axis1": [ 1152, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 37, 4 ], "axis1": [ 1152, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 42, 2 ], "axis1": [ 1152, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 68, 1 ], "axis1": [ 1152, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 70, 1 ], "axis1": [ 1152, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 81, 1 ], "axis1": [ 1152, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 84, 1 ], "axis1": [ 1152, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 92, 26 ], "axis1": [ 1152, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 120, 26 ], "axis1": [ 1152, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1152, 64 ], "axis1": [ 34, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 37, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 42, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 68, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 70, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 81, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 84, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 92, 26 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 120, 26 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 146, 64 ], "axis1": [ 1216, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1216, 64 ], "axis1": [ 146, 64 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 210, 64 ], "axis1": [ 1280, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1280, 64 ], "axis1": [ 210, 64 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 274, 64 ], "axis1": [ 1344, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1344, 64 ], "axis1": [ 274, 64 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 338, 15 ], "axis1": [ 1408, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 419, 49 ], "axis1": [ 1408, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1408, 64 ], "axis1": [ 338, 15 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1408, 64 ], "axis1": [ 419, 49 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 468, 64 ], "axis1": [ 1472, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1472, 64 ], "axis1": [ 468, 64 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 532, 64 ], "axis1": [ 1536, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1536, 64 ], "axis1": [ 532, 64 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 596, 64 ], "axis1": [ 1600, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1600, 64 ], "axis1": [ 596, 64 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 660, 9 ], "axis1": [ 1664, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 702, 15 ], "axis1": [ 1664, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 762, 33 ], "axis1": [ 1664, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 828, 6 ], "axis1": [ 1664, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 836, 1 ], "axis1": [ 1664, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1664, 64 ], "axis1": [ 660, 9 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 702, 15 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 762, 33 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 828, 6 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 836, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 837, 3 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 841, 4 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 847, 3 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 851, 1 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 853, 2 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 856, 4 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 862, 15 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 878, 1 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 881, 7 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 929, 4 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 966, 16 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 983, 4 ], "axis1": [ 1728, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1728, 64 ], "axis1": [ 837, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 841, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 847, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 851, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 853, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 856, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 862, 15 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 878, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 881, 7 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 929, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 966, 16 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 983, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1792, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1792, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 987, 1 ], "axis1": [ 1792, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 993, 27 ], "axis1": [ 1792, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1021, 2 ], "axis1": [ 1792, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1024, 17 ], "axis1": [ 1792, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1042, 4 ], "axis1": [ 1792, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1047, 4 ], "axis1": [ 1792, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1052, 9 ], "axis1": [ 1792, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1792, 64 ], "axis1": [ 987, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 993, 27 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 1021, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 1024, 17 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 1042, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 1047, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 1052, 9 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1856, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1856, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 1062, 1 ], "axis1": [ 1856, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1064, 17 ], "axis1": [ 1856, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1082, 29 ], "axis1": [ 1856, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1112, 1 ], "axis1": [ 1856, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1187, 3 ], "axis1": [ 1856, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1561, 13 ], "axis1": [ 1856, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1856, 64 ], "axis1": [ 1062, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 1064, 17 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 1082, 29 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 1112, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 1187, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 1561, 13 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 1574, 4 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1586, 1 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1636, 2 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1639, 2 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1642, 16 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1677, 1 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1698, 1 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1701, 1 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1703, 1 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1706, 1 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1709, 3 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1750, 13 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1800, 4 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1845, 14 ], "axis1": [ 1920, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1920, 64 ], "axis1": [ 1574, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1586, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1636, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1639, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1642, 16 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1677, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1698, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1701, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1703, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1706, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1709, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1750, 13 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1800, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1845, 14 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1984, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1984, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 1859, 18 ], "axis1": [ 1984, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1943, 33 ], "axis1": [ 1984, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 2045, 1 ], "axis1": [ 1984, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 1984, 64 ], "axis1": [ 1859, 18 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1984, 64 ], "axis1": [ 1943, 33 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 1984, 64 ], "axis1": [ 2045, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 34, 7 ], "axis1": [ 448, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 42, 3 ], "axis1": [ 448, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 46, 9 ], "axis1": [ 448, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 58, 1 ], "axis1": [ 448, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 61, 3 ], "axis1": [ 448, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 65, 41 ], "axis1": [ 448, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 34, 7 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 42, 3 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 46, 9 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 58, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 61, 3 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 65, 41 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 106, 12 ], "axis1": [ 512, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 119, 52 ], "axis1": [ 512, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 106, 12 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 119, 52 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 171, 64 ], "axis1": [ 576, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 171, 64 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 235, 64 ], "axis1": [ 640, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 235, 64 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 299, 54 ], "axis1": [ 704, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 419, 10 ], "axis1": [ 704, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 299, 54 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 419, 10 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 429, 15 ], "axis1": [ 768, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 477, 3 ], "axis1": [ 768, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 669, 33 ], "axis1": [ 768, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1018, 2 ], "axis1": [ 768, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1021, 2 ], "axis1": [ 768, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1024, 1 ], "axis1": [ 768, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1028, 2 ], "axis1": [ 768, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1081, 1 ], "axis1": [ 768, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1179, 5 ], "axis1": [ 768, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 429, 15 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 477, 3 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 669, 33 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1018, 2 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1021, 2 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1024, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1028, 2 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1081, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1179, 5 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 1578, 8 ], "axis1": [ 832, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1587, 48 ], "axis1": [ 832, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1641, 1 ], "axis1": [ 832, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1700, 1 ], "axis1": [ 832, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1804, 1 ], "axis1": [ 832, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1808, 1 ], "axis1": [ 832, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1810, 1 ], "axis1": [ 832, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1812, 1 ], "axis1": [ 832, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 1838, 2 ], "axis1": [ 832, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 1578, 8 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1587, 48 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1641, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1700, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1804, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1808, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1810, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1812, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1838, 2 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 1842, 1 ], "axis1": [ 896, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 1842, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 1, 34 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 64, 1 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 118, 1 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 172, 1 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 223, 12 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 237, 4 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 247, 11 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 1, 34 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 64, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 118, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 172, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 223, 12 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 237, 4 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 247, 11 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 258, 64 ], "axis1": [ 256, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 258, 64 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 322, 8 ], "axis1": [ 320, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 332, 11 ], "axis1": [ 320, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 444, 33 ], "axis1": [ 320, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 480, 12 ], "axis1": [ 320, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 322, 8 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 332, 11 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 444, 33 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 480, 12 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 492, 24 ], "axis1": [ 384, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 869, 8 ], "axis1": [ 384, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 878, 1 ], "axis1": [ 384, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 881, 7 ], "axis1": [ 384, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 929, 4 ], "axis1": [ 384, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 966, 16 ], "axis1": [ 384, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 983, 4 ], "axis1": [ 384, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 492, 24 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 869, 8 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 878, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 881, 7 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 929, 4 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 966, 16 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 983, 4 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 987, 1 ], "axis1": [ 448, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 993, 25 ], "axis1": [ 448, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 1030, 11 ], "axis1": [ 448, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 1042, 4 ], "axis1": [ 448, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 1047, 4 ], "axis1": [ 448, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 1052, 9 ], "axis1": [ 448, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 1062, 1 ], "axis1": [ 448, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 1064, 9 ], "axis1": [ 448, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 987, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 993, 25 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1030, 11 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1042, 4 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1047, 4 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1052, 9 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1062, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1064, 9 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 1073, 8 ], "axis1": [ 512, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 1082, 29 ], "axis1": [ 512, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 1112, 1 ], "axis1": [ 512, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 1187, 3 ], "axis1": [ 512, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 1223, 20 ], "axis1": [ 512, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 1357, 1 ], "axis1": [ 512, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 1382, 1 ], "axis1": [ 512, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 1690, 1 ], "axis1": [ 512, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 1073, 8 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1082, 29 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1112, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1187, 3 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1223, 20 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1357, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1382, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1690, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 1691, 3 ], "axis1": [ 576, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 1691, 3 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 34, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 41, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 64, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 130, 3 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 140, 3 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 152, 6 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 165, 3 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 169, 2 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 172, 2 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 175, 2 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 185, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 192, 3 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 196, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 198, 4 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 203, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 223, 12 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 254, 18 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 34, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 41, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 64, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 130, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 140, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 152, 6 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 165, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 169, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 172, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 175, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 185, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 192, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 196, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 198, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 203, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 223, 12 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 254, 18 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 272, 55 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 328, 1 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 331, 4 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 336, 4 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 272, 55 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 328, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 331, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 336, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 340, 3 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 344, 2 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 669, 33 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 933, 26 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 340, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 344, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 669, 33 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 933, 26 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 959, 7 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 1018, 2 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 1025, 3 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 1113, 52 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 959, 7 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 1018, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 1025, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 1113, 52 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 1165, 14 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 1184, 3 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 1244, 33 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 1635, 1 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 1724, 2 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 1877, 11 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 1165, 14 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1184, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1244, 33 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1635, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1724, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1877, 11 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 1888, 22 ], "axis1": [ 512, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 1888, 22 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 34, 11 ], "axis1": [ 512, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 46, 9 ], "axis1": [ 512, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 58, 1 ], "axis1": [ 512, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 61, 31 ], "axis1": [ 512, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 109, 6 ], "axis1": [ 512, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 118, 2 ], "axis1": [ 512, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 133, 4 ], "axis1": [ 512, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 34, 11 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 46, 9 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 58, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 61, 31 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 109, 6 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 118, 2 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 133, 4 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 137, 3 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 143, 9 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 158, 7 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 168, 1 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 171, 4 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 177, 8 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 186, 6 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 195, 3 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 199, 1 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 222, 13 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 254, 9 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 137, 3 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 143, 9 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 158, 7 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 168, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 171, 4 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 177, 8 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 186, 6 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 195, 3 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 199, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 222, 13 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 254, 9 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 263, 58 ], "axis1": [ 640, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 322, 1 ], "axis1": [ 640, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 324, 3 ], "axis1": [ 640, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 328, 2 ], "axis1": [ 640, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 263, 58 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 322, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 324, 3 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 328, 2 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 331, 4 ], "axis1": [ 704, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 336, 3 ], "axis1": [ 704, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 480, 36 ], "axis1": [ 704, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 675, 9 ], "axis1": [ 704, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 869, 8 ], "axis1": [ 704, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 878, 1 ], "axis1": [ 704, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 881, 3 ], "axis1": [ 704, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 331, 4 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 336, 3 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 480, 36 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 675, 9 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 869, 8 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 878, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 881, 3 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 884, 4 ], "axis1": [ 768, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 930, 1 ], "axis1": [ 768, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 966, 6 ], "axis1": [ 768, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 975, 4 ], "axis1": [ 768, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 1008, 1 ], "axis1": [ 768, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 1012, 1 ], "axis1": [ 768, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 1111, 1 ], "axis1": [ 768, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 1150, 1 ], "axis1": [ 768, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 1386, 1 ], "axis1": [ 768, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 884, 4 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 930, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 966, 6 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 975, 4 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1008, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1012, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1111, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1150, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 1386, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 34, 1 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 36, 1 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 39, 1 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 41, 1 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 43, 3 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 49, 9 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 59, 2 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 67, 1 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 72, 12 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 130, 3 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 143, 9 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 158, 7 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 168, 1 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 171, 1 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 173, 2 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 177, 9 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 192, 1 ], "axis1": [ 1216, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1216, 64 ], "axis1": [ 34, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 36, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 39, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 41, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 43, 3 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 49, 9 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 59, 2 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 67, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 72, 12 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 130, 3 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 143, 9 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 158, 7 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 168, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 171, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 173, 2 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 177, 9 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 192, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 195, 3 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 199, 3 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 222, 6 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 229, 3 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 245, 1 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 254, 30 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 286, 4 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 332, 1 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 339, 4 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 344, 2 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 669, 6 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 868, 1 ], "axis1": [ 1280, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1280, 64 ], "axis1": [ 195, 3 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 199, 3 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 222, 6 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 229, 3 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 245, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 254, 30 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 286, 4 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 332, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 339, 4 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 344, 2 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 669, 6 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 868, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 960, 5 ], "axis1": [ 1344, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 990, 1 ], "axis1": [ 1344, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1025, 1 ], "axis1": [ 1344, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1150, 1 ], "axis1": [ 1344, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 1344, 64 ], "axis1": [ 960, 5 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 990, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1025, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1150, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 35, 1 ], "axis1": [ 640, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 37, 2 ], "axis1": [ 640, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 40, 1 ], "axis1": [ 640, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 52, 1 ], "axis1": [ 640, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 61, 6 ], "axis1": [ 640, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 69, 1 ], "axis1": [ 640, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 72, 3 ], "axis1": [ 640, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 76, 2 ], "axis1": [ 640, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 79, 4 ], "axis1": [ 640, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 94, 36 ], "axis1": [ 640, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 133, 7 ], "axis1": [ 640, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 35, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 37, 2 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 40, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 52, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 61, 6 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 69, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 72, 3 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 76, 2 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 79, 4 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 94, 36 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 133, 7 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 186, 6 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 212, 1 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 228, 1 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 232, 3 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 241, 1 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 294, 1 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 297, 10 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 308, 4 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 313, 4 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 318, 3 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 322, 1 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 324, 5 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 330, 2 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 333, 2 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 346, 1 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 444, 19 ], "axis1": [ 704, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 186, 6 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 212, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 228, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 232, 3 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 241, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 294, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 297, 10 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 308, 4 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 313, 4 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 318, 3 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 322, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 324, 5 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 330, 2 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 333, 2 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 346, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 444, 19 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 463, 17 ], "axis1": [ 768, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 496, 18 ], "axis1": [ 768, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 515, 29 ], "axis1": [ 768, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 463, 17 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 496, 18 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 515, 29 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 544, 5 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 675, 9 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 869, 8 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 878, 1 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 883, 5 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 930, 1 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 965, 17 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 984, 2 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1006, 4 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1030, 1 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1084, 3 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1088, 1 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1243, 1 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1487, 6 ], "axis1": [ 832, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 544, 5 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 675, 9 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 869, 8 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 878, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 883, 5 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 930, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 965, 17 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 984, 2 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1006, 4 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1030, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1084, 3 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1088, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1243, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 1487, 6 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 1493, 27 ], "axis1": [ 896, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1554, 4 ], "axis1": [ 896, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1559, 29 ], "axis1": [ 896, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1822, 1 ], "axis1": [ 896, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 1824, 3 ], "axis1": [ 896, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 1493, 27 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 1554, 4 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 1559, 29 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 1822, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 1824, 3 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 1828, 3 ], "axis1": [ 960, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 960, 64 ], "axis1": [ 1828, 3 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 34, 15 ], "axis1": [ 256, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 52, 1 ], "axis1": [ 256, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 55, 23 ], "axis1": [ 256, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 79, 13 ], "axis1": [ 256, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 94, 12 ], "axis1": [ 256, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 34, 15 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 52, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 55, 23 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 79, 13 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 94, 12 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 130, 3 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 143, 9 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 158, 7 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 168, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 171, 2 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 174, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 177, 9 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 192, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 195, 7 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 203, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 222, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 229, 2 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 237, 4 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 244, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 274, 3 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 278, 4 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 283, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 286, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 288, 4 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 293, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 296, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 130, 3 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 143, 9 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 158, 7 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 168, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 171, 2 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 174, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 177, 9 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 192, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 195, 7 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 203, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 222, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 229, 2 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 237, 4 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 244, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 274, 3 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 278, 4 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 283, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 286, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 288, 4 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 293, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 296, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 329, 1 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 332, 1 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 335, 8 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 344, 2 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 480, 16 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 669, 6 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 881, 2 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 959, 6 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 982, 2 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 986, 13 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1000, 6 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1010, 1 ], "axis1": [ 384, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 329, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 332, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 335, 8 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 344, 2 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 480, 16 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 669, 6 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 881, 2 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 959, 6 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 982, 2 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 986, 13 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 1000, 6 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 1010, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 1011, 9 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1025, 1 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1031, 7 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1042, 1 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1049, 2 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1089, 3 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1093, 4 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1149, 1 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1170, 9 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1184, 6 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1223, 3 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1290, 18 ], "axis1": [ 448, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 1011, 9 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1025, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1031, 7 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1042, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1049, 2 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1089, 3 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1093, 4 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1149, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1170, 9 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1184, 6 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1223, 3 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 1290, 18 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 1308, 6 ], "axis1": [ 512, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1332, 1 ], "axis1": [ 512, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1357, 1 ], "axis1": [ 512, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1369, 1 ], "axis1": [ 512, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1380, 6 ], "axis1": [ 512, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1569, 12 ], "axis1": [ 512, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1588, 37 ], "axis1": [ 512, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 1308, 6 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1332, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1357, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1369, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1380, 6 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1569, 12 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 1588, 37 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 1625, 13 ], "axis1": [ 576, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1639, 6 ], "axis1": [ 576, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1827, 1 ], "axis1": [ 576, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1833, 2 ], "axis1": [ 576, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 1854, 42 ], "axis1": [ 576, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 1625, 13 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 1639, 6 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 1827, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 1833, 2 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 1854, 42 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 1896, 57 ], "axis1": [ 640, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 1896, 57 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 34, 64 ], "axis1": [ 768, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 34, 64 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 98, 42 ], "axis1": [ 832, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 143, 9 ], "axis1": [ 832, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 158, 7 ], "axis1": [ 832, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 168, 1 ], "axis1": [ 832, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 171, 4 ], "axis1": [ 832, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 177, 1 ], "axis1": [ 832, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 98, 42 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 143, 9 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 158, 7 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 168, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 171, 4 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 177, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 178, 10 ], "axis1": [ 896, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 192, 1 ], "axis1": [ 896, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 195, 7 ], "axis1": [ 896, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 203, 1 ], "axis1": [ 896, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 222, 6 ], "axis1": [ 896, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 229, 3 ], "axis1": [ 896, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 237, 4 ], "axis1": [ 896, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 244, 3 ], "axis1": [ 896, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 251, 29 ], "axis1": [ 896, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 178, 10 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 192, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 195, 7 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 203, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 222, 6 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 229, 3 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 237, 4 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 244, 3 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 251, 29 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 280, 14 ], "axis1": [ 960, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 296, 1 ], "axis1": [ 960, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 321, 1 ], "axis1": [ 960, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 323, 48 ], "axis1": [ 960, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 960, 64 ], "axis1": [ 280, 14 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 960, 64 ], "axis1": [ 296, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 960, 64 ], "axis1": [ 321, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 960, 64 ], "axis1": [ 323, 48 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 371, 64 ], "axis1": [ 1024, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1024, 64 ], "axis1": [ 371, 64 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 435, 64 ], "axis1": [ 1088, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1088, 64 ], "axis1": [ 435, 64 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 499, 50 ], "axis1": [ 1152, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 669, 7 ], "axis1": [ 1152, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 868, 1 ], "axis1": [ 1152, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 881, 2 ], "axis1": [ 1152, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 959, 4 ], "axis1": [ 1152, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1152, 64 ], "axis1": [ 499, 50 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 669, 7 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 868, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 881, 2 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 959, 4 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 963, 2 ], "axis1": [ 1216, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 982, 2 ], "axis1": [ 1216, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 986, 13 ], "axis1": [ 1216, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1000, 6 ], "axis1": [ 1216, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1010, 11 ], "axis1": [ 1216, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1023, 1 ], "axis1": [ 1216, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1025, 3 ], "axis1": [ 1216, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1031, 15 ], "axis1": [ 1216, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1047, 4 ], "axis1": [ 1216, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1052, 7 ], "axis1": [ 1216, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1216, 64 ], "axis1": [ 963, 2 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 982, 2 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 986, 13 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1000, 6 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1010, 11 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1023, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1025, 3 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1031, 15 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1047, 4 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 1052, 7 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 1059, 1 ], "axis1": [ 1280, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1065, 16 ], "axis1": [ 1280, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1082, 2 ], "axis1": [ 1280, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1087, 1 ], "axis1": [ 1280, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1089, 44 ], "axis1": [ 1280, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1280, 64 ], "axis1": [ 1059, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 1065, 16 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 1082, 2 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 1087, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 1089, 44 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 1133, 13 ], "axis1": [ 1344, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1147, 2 ], "axis1": [ 1344, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1150, 1 ], "axis1": [ 1344, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1152, 27 ], "axis1": [ 1344, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1184, 6 ], "axis1": [ 1344, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1223, 3 ], "axis1": [ 1344, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1238, 1 ], "axis1": [ 1344, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1247, 2 ], "axis1": [ 1344, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1253, 1 ], "axis1": [ 1344, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1257, 8 ], "axis1": [ 1344, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1344, 64 ], "axis1": [ 1133, 13 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1147, 2 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1150, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1152, 27 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1184, 6 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1223, 3 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1238, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1247, 2 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1253, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 1257, 8 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 1265, 37 ], "axis1": [ 1408, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1303, 1 ], "axis1": [ 1408, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1305, 26 ], "axis1": [ 1408, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1408, 64 ], "axis1": [ 1265, 37 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1408, 64 ], "axis1": [ 1303, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1408, 64 ], "axis1": [ 1305, 26 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 1331, 64 ], "axis1": [ 1472, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1472, 64 ], "axis1": [ 1331, 64 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 1395, 4 ], "axis1": [ 1536, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1520, 15 ], "axis1": [ 1536, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1536, 18 ], "axis1": [ 1536, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1643, 1 ], "axis1": [ 1536, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1645, 26 ], "axis1": [ 1536, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1536, 64 ], "axis1": [ 1395, 4 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1520, 15 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1536, 18 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1643, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 1645, 26 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 1671, 8 ], "axis1": [ 1600, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1680, 56 ], "axis1": [ 1600, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1600, 64 ], "axis1": [ 1671, 8 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 1680, 56 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 1736, 54 ], "axis1": [ 1664, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1791, 10 ], "axis1": [ 1664, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1664, 64 ], "axis1": [ 1736, 54 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 1791, 10 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1728, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1728, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 1801, 26 ], "axis1": [ 1728, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1828, 5 ], "axis1": [ 1728, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1835, 6 ], "axis1": [ 1728, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1842, 12 ], "axis1": [ 1728, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1855, 1 ], "axis1": [ 1728, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1857, 2 ], "axis1": [ 1728, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1876, 1 ], "axis1": [ 1728, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1878, 11 ], "axis1": [ 1728, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1728, 64 ], "axis1": [ 1801, 26 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1828, 5 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1835, 6 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1842, 12 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1855, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1857, 2 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1876, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1878, 11 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1792, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1792, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 1903, 3 ], "axis1": [ 1792, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1953, 33 ], "axis1": [ 1792, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 1792, 64 ], "axis1": [ 1903, 3 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 1953, 33 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 34, 64 ], "axis1": [ 896, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 34, 64 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 98, 64 ], "axis1": [ 960, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 960, 64 ], "axis1": [ 98, 64 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 162, 64 ], "axis1": [ 1024, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1024, 64 ], "axis1": [ 162, 64 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 226, 64 ], "axis1": [ 1088, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1088, 64 ], "axis1": [ 226, 64 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 290, 64 ], "axis1": [ 1152, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1152, 64 ], "axis1": [ 290, 64 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 354, 64 ], "axis1": [ 1216, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1216, 64 ], "axis1": [ 354, 64 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 418, 64 ], "axis1": [ 1280, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1280, 64 ], "axis1": [ 418, 64 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 482, 57 ], "axis1": [ 1344, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 549, 7 ], "axis1": [ 1344, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1344, 64 ], "axis1": [ 482, 57 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 549, 7 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 556, 64 ], "axis1": [ 1408, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1408, 64 ], "axis1": [ 556, 64 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 620, 49 ], "axis1": [ 1472, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 684, 15 ], "axis1": [ 1472, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1472, 64 ], "axis1": [ 620, 49 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 684, 15 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 699, 64 ], "axis1": [ 1536, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1536, 64 ], "axis1": [ 699, 64 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 763, 64 ], "axis1": [ 1600, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1600, 64 ], "axis1": [ 763, 64 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 827, 41 ], "axis1": [ 1664, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 877, 1 ], "axis1": [ 1664, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 879, 2 ], "axis1": [ 1664, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 888, 20 ], "axis1": [ 1664, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1664, 64 ], "axis1": [ 827, 41 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 877, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 879, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 888, 20 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 908, 22 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 931, 28 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 999, 1 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1021, 2 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1024, 1 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1028, 2 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1051, 1 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1063, 1 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1081, 1 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1146, 1 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1149, 1 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1151, 1 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1179, 2 ], "axis1": [ 1728, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1728, 64 ], "axis1": [ 908, 22 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 931, 28 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 999, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1021, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1024, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1028, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1051, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1063, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1081, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1146, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1149, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1151, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 1179, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1792, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1792, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 1181, 3 ], "axis1": [ 1792, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1190, 33 ], "axis1": [ 1792, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1302, 1 ], "axis1": [ 1792, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1304, 1 ], "axis1": [ 1792, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1399, 26 ], "axis1": [ 1792, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1792, 64 ], "axis1": [ 1181, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 1190, 33 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 1302, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 1304, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 1399, 26 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1856, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1856, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 1425, 7 ], "axis1": [ 1856, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1433, 24 ], "axis1": [ 1856, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1458, 29 ], "axis1": [ 1856, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1558, 1 ], "axis1": [ 1856, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1569, 3 ], "axis1": [ 1856, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1856, 64 ], "axis1": [ 1425, 7 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 1433, 24 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 1458, 29 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 1558, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 1569, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1920, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1920, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 1572, 41 ], "axis1": [ 1920, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1841, 1 ], "axis1": [ 1920, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1913, 1 ], "axis1": [ 1920, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1986, 21 ], "axis1": [ 1920, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1920, 64 ], "axis1": [ 1572, 41 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1841, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1913, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 1986, 21 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1984, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1984, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 2007, 39 ], "axis1": [ 1984, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 1984, 64 ], "axis1": [ 2007, 39 ], "matrix": "L17.attn.W_O" } ], "physical:cancel_bypass": [ { "axis0": [ 0, 1 ], "axis1": [ 76, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 70, 1 ], "axis1": [ 76, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 70, 1 ], "axis1": [ 77, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 76, 1 ], "axis1": [ 70, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 77, 1 ], "axis1": [ 70, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 78, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 71, 1 ], "axis1": [ 78, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 71, 1 ], "axis1": [ 79, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 78, 1 ], "axis1": [ 71, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 79, 1 ], "axis1": [ 71, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 80, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 72, 1 ], "axis1": [ 80, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 72, 1 ], "axis1": [ 81, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 80, 1 ], "axis1": [ 72, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 81, 1 ], "axis1": [ 72, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 82, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 82, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 83, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 82, 1 ], "axis1": [ 42, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 83, 1 ], "axis1": [ 42, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 84, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 61, 1 ], "axis1": [ 84, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 61, 1 ], "axis1": [ 85, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 84, 1 ], "axis1": [ 61, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 85, 1 ], "axis1": [ 61, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 86, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 62, 1 ], "axis1": [ 86, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 62, 1 ], "axis1": [ 87, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 62, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 87, 1 ], "axis1": [ 62, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 88, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 63, 1 ], "axis1": [ 88, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 63, 1 ], "axis1": [ 89, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 63, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 89, 1 ], "axis1": [ 63, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 90, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 153, 1 ], "axis1": [ 90, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 153, 1 ], "axis1": [ 91, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 153, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 91, 1 ], "axis1": [ 153, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 92, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 154, 1 ], "axis1": [ 92, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 154, 1 ], "axis1": [ 93, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 154, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 93, 1 ], "axis1": [ 154, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 94, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 133, 1 ], "axis1": [ 94, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 133, 1 ], "axis1": [ 95, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 133, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 95, 1 ], "axis1": [ 133, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 96, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 134, 1 ], "axis1": [ 96, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 134, 1 ], "axis1": [ 97, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 134, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 97, 1 ], "axis1": [ 134, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 98, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 34, 1 ], "axis1": [ 98, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 34, 1 ], "axis1": [ 99, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 34, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 99, 1 ], "axis1": [ 34, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 100, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 120, 1 ], "axis1": [ 100, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 120, 1 ], "axis1": [ 101, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 120, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 101, 1 ], "axis1": [ 120, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 102, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 121, 1 ], "axis1": [ 102, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 121, 1 ], "axis1": [ 103, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 102, 1 ], "axis1": [ 121, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 103, 1 ], "axis1": [ 121, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 104, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 122, 1 ], "axis1": [ 104, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 122, 1 ], "axis1": [ 105, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 104, 1 ], "axis1": [ 122, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 105, 1 ], "axis1": [ 122, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 106, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 106, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 107, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 151, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 107, 1 ], "axis1": [ 151, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 108, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 123, 1 ], "axis1": [ 108, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 123, 1 ], "axis1": [ 109, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 108, 1 ], "axis1": [ 123, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 109, 1 ], "axis1": [ 123, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 110, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 152, 1 ], "axis1": [ 110, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 111, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 110, 1 ], "axis1": [ 152, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 111, 1 ], "axis1": [ 152, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 112, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 128, 1 ], "axis1": [ 112, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 128, 1 ], "axis1": [ 113, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 112, 1 ], "axis1": [ 128, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 113, 1 ], "axis1": [ 128, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 114, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 135, 1 ], "axis1": [ 114, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 135, 1 ], "axis1": [ 115, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 114, 1 ], "axis1": [ 135, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 115, 1 ], "axis1": [ 135, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 116, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 116, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 117, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 116, 1 ], "axis1": [ 136, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 117, 1 ], "axis1": [ 136, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 118, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 129, 1 ], "axis1": [ 118, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 129, 1 ], "axis1": [ 119, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 118, 1 ], "axis1": [ 129, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 119, 1 ], "axis1": [ 129, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 120, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 125, 1 ], "axis1": [ 120, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 125, 1 ], "axis1": [ 121, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 120, 1 ], "axis1": [ 125, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 121, 1 ], "axis1": [ 125, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 122, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 126, 1 ], "axis1": [ 122, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 126, 1 ], "axis1": [ 123, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 122, 1 ], "axis1": [ 126, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 123, 1 ], "axis1": [ 126, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 124, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 131, 1 ], "axis1": [ 124, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 131, 1 ], "axis1": [ 125, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 124, 1 ], "axis1": [ 131, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 125, 1 ], "axis1": [ 131, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 126, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 127, 1 ], "axis1": [ 126, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 127, 1 ], "axis1": [ 127, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 126, 1 ], "axis1": [ 127, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 127, 1 ], "axis1": [ 127, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 132, 1 ], "axis1": [ 128, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 132, 1 ], "axis1": [ 129, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 128, 1 ], "axis1": [ 132, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 129, 1 ], "axis1": [ 132, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 130, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 155, 1 ], "axis1": [ 130, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 155, 1 ], "axis1": [ 131, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 130, 1 ], "axis1": [ 155, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 131, 1 ], "axis1": [ 155, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 132, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 156, 1 ], "axis1": [ 132, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 156, 1 ], "axis1": [ 133, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 132, 1 ], "axis1": [ 156, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 133, 1 ], "axis1": [ 156, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 134, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 157, 1 ], "axis1": [ 134, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 157, 1 ], "axis1": [ 135, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 134, 1 ], "axis1": [ 157, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 135, 1 ], "axis1": [ 157, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 136, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 158, 1 ], "axis1": [ 136, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 158, 1 ], "axis1": [ 137, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 158, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 137, 1 ], "axis1": [ 158, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 138, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 159, 1 ], "axis1": [ 138, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 159, 1 ], "axis1": [ 139, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 138, 1 ], "axis1": [ 159, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 139, 1 ], "axis1": [ 159, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 140, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 140, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 141, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 140, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 141, 1 ], "axis1": [ 140, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 142, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 142, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 143, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 141, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 143, 1 ], "axis1": [ 141, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 144, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 144, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 145, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 142, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 145, 1 ], "axis1": [ 142, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 146, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 146, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 147, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 143, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 147, 1 ], "axis1": [ 143, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 148, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 148, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 149, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 144, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 149, 1 ], "axis1": [ 144, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 150, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 828, 1 ], "axis1": [ 150, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 828, 1 ], "axis1": [ 151, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 828, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 151, 1 ], "axis1": [ 828, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 152, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 829, 1 ], "axis1": [ 152, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 829, 1 ], "axis1": [ 153, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 152, 1 ], "axis1": [ 829, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 153, 1 ], "axis1": [ 829, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 154, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 830, 1 ], "axis1": [ 154, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 830, 1 ], "axis1": [ 155, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 154, 1 ], "axis1": [ 830, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 155, 1 ], "axis1": [ 830, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 157, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 1805, 1 ], "axis1": [ 157, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1805, 1 ], "axis1": [ 158, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 157, 1 ], "axis1": [ 1805, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 158, 1 ], "axis1": [ 1805, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 159, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 1807, 1 ], "axis1": [ 159, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1807, 1 ], "axis1": [ 160, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 159, 1 ], "axis1": [ 1807, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 160, 1 ], "axis1": [ 1807, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 161, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 1813, 1 ], "axis1": [ 161, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1813, 1 ], "axis1": [ 162, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 161, 1 ], "axis1": [ 1813, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 162, 1 ], "axis1": [ 1813, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 163, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 1814, 1 ], "axis1": [ 163, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1814, 1 ], "axis1": [ 164, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 163, 1 ], "axis1": [ 1814, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 164, 1 ], "axis1": [ 1814, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 165, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 1815, 1 ], "axis1": [ 165, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1815, 1 ], "axis1": [ 166, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 165, 1 ], "axis1": [ 1815, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 166, 1 ], "axis1": [ 1815, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 167, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 1816, 1 ], "axis1": [ 167, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1816, 1 ], "axis1": [ 168, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 167, 1 ], "axis1": [ 1816, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 168, 1 ], "axis1": [ 1816, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 169, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 1793, 1 ], "axis1": [ 169, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1793, 1 ], "axis1": [ 170, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 169, 1 ], "axis1": [ 1793, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 170, 1 ], "axis1": [ 1793, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 171, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 47, 1 ], "axis1": [ 171, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 47, 1 ], "axis1": [ 172, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 171, 1 ], "axis1": [ 47, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 172, 1 ], "axis1": [ 47, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 173, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 1820, 1 ], "axis1": [ 173, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1820, 1 ], "axis1": [ 174, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 1820, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 174, 1 ], "axis1": [ 1820, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 175, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 175, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 176, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 175, 1 ], "axis1": [ 139, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 176, 1 ], "axis1": [ 139, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 177, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 1821, 1 ], "axis1": [ 177, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1821, 1 ], "axis1": [ 178, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 177, 1 ], "axis1": [ 1821, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 178, 1 ], "axis1": [ 1821, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 179, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 1809, 1 ], "axis1": [ 179, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1809, 1 ], "axis1": [ 180, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 179, 1 ], "axis1": [ 1809, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 180, 1 ], "axis1": [ 1809, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 181, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 1811, 1 ], "axis1": [ 181, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1811, 1 ], "axis1": [ 182, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 181, 1 ], "axis1": [ 1811, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 182, 1 ], "axis1": [ 1811, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 183, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 1798, 1 ], "axis1": [ 183, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1798, 1 ], "axis1": [ 184, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 183, 1 ], "axis1": [ 1798, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 184, 1 ], "axis1": [ 1798, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 185, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 1799, 1 ], "axis1": [ 185, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1799, 1 ], "axis1": [ 186, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 185, 1 ], "axis1": [ 1799, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 186, 1 ], "axis1": [ 1799, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 187, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 48, 1 ], "axis1": [ 187, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 48, 1 ], "axis1": [ 188, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 187, 1 ], "axis1": [ 48, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 188, 1 ], "axis1": [ 48, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 310, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 983, 1 ], "axis1": [ 310, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 983, 1 ], "axis1": [ 311, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 310, 1 ], "axis1": [ 983, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 311, 1 ], "axis1": [ 983, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 312, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 984, 1 ], "axis1": [ 312, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 984, 1 ], "axis1": [ 313, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 312, 1 ], "axis1": [ 984, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 313, 1 ], "axis1": [ 984, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 928, 66 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 44, 1 ], "axis1": [ 928, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1085, 26 ], "axis1": [ 928, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1112, 6 ], "axis1": [ 928, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 44, 1 ], "axis1": [ 961, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1085, 26 ], "axis1": [ 961, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1112, 6 ], "axis1": [ 961, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 928, 1 ], "axis1": [ 44, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 929, 26 ], "axis1": [ 1085, 26 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 955, 6 ], "axis1": [ 1112, 6 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 961, 1 ], "axis1": [ 44, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 962, 26 ], "axis1": [ 1085, 26 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 988, 6 ], "axis1": [ 1112, 6 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 994, 66 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 168, 33 ], "axis1": [ 994, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 168, 33 ], "axis1": [ 1027, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 994, 33 ], "axis1": [ 168, 33 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1027, 33 ], "axis1": [ 168, 33 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1060, 66 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 234, 33 ], "axis1": [ 1060, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 234, 33 ], "axis1": [ 1093, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1060, 33 ], "axis1": [ 234, 33 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1093, 33 ], "axis1": [ 234, 33 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1126, 66 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 300, 33 ], "axis1": [ 1126, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 300, 33 ], "axis1": [ 1159, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1126, 33 ], "axis1": [ 300, 33 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1159, 33 ], "axis1": [ 300, 33 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1192, 66 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 366, 33 ], "axis1": [ 1192, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 366, 33 ], "axis1": [ 1225, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1192, 33 ], "axis1": [ 366, 33 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1225, 33 ], "axis1": [ 366, 33 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1258, 66 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 921, 9 ], "axis1": [ 1258, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 931, 4 ], "axis1": [ 1258, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 936, 4 ], "axis1": [ 1258, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 941, 4 ], "axis1": [ 1258, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 946, 4 ], "axis1": [ 1258, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 951, 3 ], "axis1": [ 1258, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1813, 5 ], "axis1": [ 1258, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 921, 9 ], "axis1": [ 1291, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 931, 4 ], "axis1": [ 1291, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 936, 4 ], "axis1": [ 1291, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 941, 4 ], "axis1": [ 1291, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 946, 4 ], "axis1": [ 1291, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 951, 3 ], "axis1": [ 1291, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1813, 5 ], "axis1": [ 1291, 33 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1258, 9 ], "axis1": [ 921, 9 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1267, 4 ], "axis1": [ 931, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1271, 4 ], "axis1": [ 936, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1275, 4 ], "axis1": [ 941, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1279, 4 ], "axis1": [ 946, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1283, 3 ], "axis1": [ 951, 3 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1286, 5 ], "axis1": [ 1813, 5 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1291, 9 ], "axis1": [ 921, 9 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1300, 4 ], "axis1": [ 931, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1304, 4 ], "axis1": [ 936, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1308, 4 ], "axis1": [ 941, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1312, 4 ], "axis1": [ 946, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1316, 3 ], "axis1": [ 951, 3 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1319, 5 ], "axis1": [ 1813, 5 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1324, 8 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 77, 4 ], "axis1": [ 1324, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 77, 4 ], "axis1": [ 1328, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1324, 4 ], "axis1": [ 77, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1328, 4 ], "axis1": [ 77, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1332, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1796, 1 ], "axis1": [ 1332, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1796, 1 ], "axis1": [ 1333, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1332, 1 ], "axis1": [ 1796, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1333, 1 ], "axis1": [ 1796, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1334, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 49, 1 ], "axis1": [ 1334, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 1335, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1334, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1335, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1336, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 53, 1 ], "axis1": [ 1336, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 53, 1 ], "axis1": [ 1337, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1336, 1 ], "axis1": [ 53, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1337, 1 ], "axis1": [ 53, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1338, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 37, 1 ], "axis1": [ 1338, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 37, 1 ], "axis1": [ 1339, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1338, 1 ], "axis1": [ 37, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1339, 1 ], "axis1": [ 37, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1340, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1072, 1 ], "axis1": [ 1340, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1072, 1 ], "axis1": [ 1341, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1340, 1 ], "axis1": [ 1072, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1341, 1 ], "axis1": [ 1072, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1342, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1005, 1 ], "axis1": [ 1342, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1005, 1 ], "axis1": [ 1343, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1342, 1 ], "axis1": [ 1005, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1343, 1 ], "axis1": [ 1005, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1006, 1 ], "axis1": [ 1344, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1006, 1 ], "axis1": [ 1345, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1344, 1 ], "axis1": [ 1006, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1345, 1 ], "axis1": [ 1006, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1346, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1007, 1 ], "axis1": [ 1346, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1007, 1 ], "axis1": [ 1347, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1346, 1 ], "axis1": [ 1007, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1347, 1 ], "axis1": [ 1007, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1348, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1018, 1 ], "axis1": [ 1348, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1018, 1 ], "axis1": [ 1349, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1348, 1 ], "axis1": [ 1018, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1349, 1 ], "axis1": [ 1018, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1350, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1019, 1 ], "axis1": [ 1350, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1019, 1 ], "axis1": [ 1351, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1350, 1 ], "axis1": [ 1019, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1351, 1 ], "axis1": [ 1019, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1352, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1025, 1 ], "axis1": [ 1352, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1025, 1 ], "axis1": [ 1353, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1352, 1 ], "axis1": [ 1025, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1353, 1 ], "axis1": [ 1025, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1354, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1832, 1 ], "axis1": [ 1354, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1832, 1 ], "axis1": [ 1355, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1354, 1 ], "axis1": [ 1832, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1355, 1 ], "axis1": [ 1832, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1356, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1027, 1 ], "axis1": [ 1356, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1027, 1 ], "axis1": [ 1357, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1356, 1 ], "axis1": [ 1027, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1357, 1 ], "axis1": [ 1027, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1358, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1029, 1 ], "axis1": [ 1358, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1029, 1 ], "axis1": [ 1359, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1358, 1 ], "axis1": [ 1029, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1359, 1 ], "axis1": [ 1029, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1360, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1030, 1 ], "axis1": [ 1360, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1030, 1 ], "axis1": [ 1361, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1360, 1 ], "axis1": [ 1030, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1361, 1 ], "axis1": [ 1030, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1362, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1031, 1 ], "axis1": [ 1362, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1031, 1 ], "axis1": [ 1363, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1362, 1 ], "axis1": [ 1031, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1363, 1 ], "axis1": [ 1031, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1364, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1032, 1 ], "axis1": [ 1364, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1032, 1 ], "axis1": [ 1365, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1364, 1 ], "axis1": [ 1032, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1365, 1 ], "axis1": [ 1032, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1366, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1034, 1 ], "axis1": [ 1366, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1034, 1 ], "axis1": [ 1367, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1366, 1 ], "axis1": [ 1034, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1367, 1 ], "axis1": [ 1034, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1368, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1035, 1 ], "axis1": [ 1368, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1035, 1 ], "axis1": [ 1369, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1368, 1 ], "axis1": [ 1035, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1369, 1 ], "axis1": [ 1035, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1370, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1036, 1 ], "axis1": [ 1370, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1036, 1 ], "axis1": [ 1371, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1370, 1 ], "axis1": [ 1036, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1371, 1 ], "axis1": [ 1036, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1372, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1037, 1 ], "axis1": [ 1372, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1037, 1 ], "axis1": [ 1373, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1372, 1 ], "axis1": [ 1037, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1373, 1 ], "axis1": [ 1037, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1374, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1042, 1 ], "axis1": [ 1374, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1042, 1 ], "axis1": [ 1375, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1374, 1 ], "axis1": [ 1042, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1375, 1 ], "axis1": [ 1042, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1376, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1043, 1 ], "axis1": [ 1376, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1043, 1 ], "axis1": [ 1377, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1376, 1 ], "axis1": [ 1043, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1377, 1 ], "axis1": [ 1043, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1378, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1049, 1 ], "axis1": [ 1378, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1049, 1 ], "axis1": [ 1379, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1378, 1 ], "axis1": [ 1049, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1379, 1 ], "axis1": [ 1049, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1380, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1052, 1 ], "axis1": [ 1380, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1052, 1 ], "axis1": [ 1381, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1380, 1 ], "axis1": [ 1052, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1381, 1 ], "axis1": [ 1052, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1382, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1053, 1 ], "axis1": [ 1382, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1053, 1 ], "axis1": [ 1383, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1382, 1 ], "axis1": [ 1053, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1383, 1 ], "axis1": [ 1053, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1384, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1056, 1 ], "axis1": [ 1384, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1056, 1 ], "axis1": [ 1385, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1384, 1 ], "axis1": [ 1056, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1385, 1 ], "axis1": [ 1056, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1386, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1057, 1 ], "axis1": [ 1386, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1057, 1 ], "axis1": [ 1387, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1386, 1 ], "axis1": [ 1057, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1387, 1 ], "axis1": [ 1057, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1388, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1060, 1 ], "axis1": [ 1388, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1060, 1 ], "axis1": [ 1389, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1388, 1 ], "axis1": [ 1060, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1389, 1 ], "axis1": [ 1060, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1390, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 1390, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 1391, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1390, 1 ], "axis1": [ 42, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1391, 1 ], "axis1": [ 42, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1392, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1062, 1 ], "axis1": [ 1392, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1062, 1 ], "axis1": [ 1393, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1392, 1 ], "axis1": [ 1062, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1393, 1 ], "axis1": [ 1062, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1394, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 1064, 1 ], "axis1": [ 1394, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1064, 1 ], "axis1": [ 1395, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1394, 1 ], "axis1": [ 1064, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 1395, 1 ], "axis1": [ 1064, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 561, 66 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 66, 1 ], "axis1": [ 561, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 135, 2 ], "axis1": [ 561, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 138, 12 ], "axis1": [ 561, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1656, 2 ], "axis1": [ 561, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1699, 1 ], "axis1": [ 561, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1763, 15 ], "axis1": [ 561, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 594, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 135, 2 ], "axis1": [ 594, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 138, 12 ], "axis1": [ 594, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1656, 2 ], "axis1": [ 594, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1699, 1 ], "axis1": [ 594, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1763, 15 ], "axis1": [ 594, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 561, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 562, 2 ], "axis1": [ 135, 2 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 564, 12 ], "axis1": [ 138, 12 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 576, 2 ], "axis1": [ 1656, 2 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 578, 1 ], "axis1": [ 1699, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 579, 15 ], "axis1": [ 1763, 15 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 594, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 595, 2 ], "axis1": [ 135, 2 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 597, 12 ], "axis1": [ 138, 12 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 609, 2 ], "axis1": [ 1656, 2 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 611, 1 ], "axis1": [ 1699, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 612, 15 ], "axis1": [ 1763, 15 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 627, 66 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 38, 2 ], "axis1": [ 627, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 164, 3 ], "axis1": [ 627, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 828, 3 ], "axis1": [ 627, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 832, 2 ], "axis1": [ 627, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 836, 4 ], "axis1": [ 627, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 841, 4 ], "axis1": [ 627, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 983, 3 ], "axis1": [ 627, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 994, 4 ], "axis1": [ 627, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1664, 4 ], "axis1": [ 627, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1792, 4 ], "axis1": [ 627, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 38, 2 ], "axis1": [ 660, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 164, 3 ], "axis1": [ 660, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 828, 3 ], "axis1": [ 660, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 832, 2 ], "axis1": [ 660, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 836, 4 ], "axis1": [ 660, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 841, 4 ], "axis1": [ 660, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 983, 3 ], "axis1": [ 660, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 994, 4 ], "axis1": [ 660, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1664, 4 ], "axis1": [ 660, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1792, 4 ], "axis1": [ 660, 33 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 627, 2 ], "axis1": [ 38, 2 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 629, 3 ], "axis1": [ 164, 3 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 632, 3 ], "axis1": [ 828, 3 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 635, 2 ], "axis1": [ 832, 2 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 637, 4 ], "axis1": [ 836, 4 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 641, 4 ], "axis1": [ 841, 4 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 645, 3 ], "axis1": [ 983, 3 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 648, 4 ], "axis1": [ 994, 4 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 652, 4 ], "axis1": [ 1664, 4 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 656, 4 ], "axis1": [ 1792, 4 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 660, 2 ], "axis1": [ 38, 2 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 662, 3 ], "axis1": [ 164, 3 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 665, 3 ], "axis1": [ 828, 3 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 668, 2 ], "axis1": [ 832, 2 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 670, 4 ], "axis1": [ 836, 4 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 674, 4 ], "axis1": [ 841, 4 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 678, 3 ], "axis1": [ 983, 3 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 681, 4 ], "axis1": [ 994, 4 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 685, 4 ], "axis1": [ 1664, 4 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 689, 4 ], "axis1": [ 1792, 4 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 693, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 43, 1 ], "axis1": [ 693, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 43, 1 ], "axis1": [ 694, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 693, 1 ], "axis1": [ 43, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 694, 1 ], "axis1": [ 43, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 695, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1689, 1 ], "axis1": [ 695, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1689, 1 ], "axis1": [ 696, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 695, 1 ], "axis1": [ 1689, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 696, 1 ], "axis1": [ 1689, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 697, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1482, 1 ], "axis1": [ 697, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1482, 1 ], "axis1": [ 698, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 697, 1 ], "axis1": [ 1482, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 698, 1 ], "axis1": [ 1482, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1029, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 935, 33 ], "axis1": [ 1029, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 935, 33 ], "axis1": [ 1062, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1029, 33 ], "axis1": [ 935, 33 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1062, 33 ], "axis1": [ 935, 33 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1095, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1028, 13 ], "axis1": [ 1095, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1042, 4 ], "axis1": [ 1095, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1047, 3 ], "axis1": [ 1095, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1052, 9 ], "axis1": [ 1095, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1062, 1 ], "axis1": [ 1095, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1064, 3 ], "axis1": [ 1095, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1028, 13 ], "axis1": [ 1128, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1042, 4 ], "axis1": [ 1128, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1047, 3 ], "axis1": [ 1128, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1052, 9 ], "axis1": [ 1128, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1062, 1 ], "axis1": [ 1128, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1064, 3 ], "axis1": [ 1128, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1095, 13 ], "axis1": [ 1028, 13 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1108, 4 ], "axis1": [ 1042, 4 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1112, 3 ], "axis1": [ 1047, 3 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1115, 9 ], "axis1": [ 1052, 9 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1124, 1 ], "axis1": [ 1062, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1125, 3 ], "axis1": [ 1064, 3 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1128, 13 ], "axis1": [ 1028, 13 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1141, 4 ], "axis1": [ 1042, 4 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1145, 3 ], "axis1": [ 1047, 3 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1148, 9 ], "axis1": [ 1052, 9 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1157, 1 ], "axis1": [ 1062, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1158, 3 ], "axis1": [ 1064, 3 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1161, 66 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1067, 1 ], "axis1": [ 1161, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1069, 4 ], "axis1": [ 1161, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1078, 1 ], "axis1": [ 1161, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1085, 26 ], "axis1": [ 1161, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1112, 1 ], "axis1": [ 1161, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1067, 1 ], "axis1": [ 1194, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1069, 4 ], "axis1": [ 1194, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1078, 1 ], "axis1": [ 1194, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1085, 26 ], "axis1": [ 1194, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1112, 1 ], "axis1": [ 1194, 33 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1161, 1 ], "axis1": [ 1067, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1162, 4 ], "axis1": [ 1069, 4 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1166, 1 ], "axis1": [ 1078, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1167, 26 ], "axis1": [ 1085, 26 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1193, 1 ], "axis1": [ 1112, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1194, 1 ], "axis1": [ 1067, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1195, 4 ], "axis1": [ 1069, 4 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1199, 1 ], "axis1": [ 1078, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1200, 26 ], "axis1": [ 1085, 26 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1226, 1 ], "axis1": [ 1112, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1227, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 323, 1 ], "axis1": [ 1227, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 323, 1 ], "axis1": [ 1228, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1227, 1 ], "axis1": [ 323, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1228, 1 ], "axis1": [ 323, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1229, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1008, 1 ], "axis1": [ 1229, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1008, 1 ], "axis1": [ 1230, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1229, 1 ], "axis1": [ 1008, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1230, 1 ], "axis1": [ 1008, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1231, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 881, 1 ], "axis1": [ 1231, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 881, 1 ], "axis1": [ 1232, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1231, 1 ], "axis1": [ 881, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1232, 1 ], "axis1": [ 881, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1233, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 884, 1 ], "axis1": [ 1233, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 884, 1 ], "axis1": [ 1234, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1233, 1 ], "axis1": [ 884, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1234, 1 ], "axis1": [ 884, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1235, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1725, 1 ], "axis1": [ 1235, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1725, 1 ], "axis1": [ 1236, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1235, 1 ], "axis1": [ 1725, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1236, 1 ], "axis1": [ 1725, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1237, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 167, 1 ], "axis1": [ 1237, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 167, 1 ], "axis1": [ 1238, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1237, 1 ], "axis1": [ 167, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1238, 1 ], "axis1": [ 167, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1239, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 351, 1 ], "axis1": [ 1239, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 351, 1 ], "axis1": [ 1240, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1239, 1 ], "axis1": [ 351, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1240, 1 ], "axis1": [ 351, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1241, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 352, 1 ], "axis1": [ 1241, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 352, 1 ], "axis1": [ 1242, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1241, 1 ], "axis1": [ 352, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 1242, 1 ], "axis1": [ 352, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 667, 66 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1223, 33 ], "axis1": [ 667, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1223, 33 ], "axis1": [ 700, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 667, 33 ], "axis1": [ 1223, 33 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 700, 33 ], "axis1": [ 1223, 33 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 733, 66 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1256, 33 ], "axis1": [ 733, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1256, 33 ], "axis1": [ 766, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 733, 33 ], "axis1": [ 1256, 33 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 766, 33 ], "axis1": [ 1256, 33 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 799, 66 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1289, 25 ], "axis1": [ 799, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1380, 2 ], "axis1": [ 799, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1383, 3 ], "axis1": [ 799, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1387, 3 ], "axis1": [ 799, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1289, 25 ], "axis1": [ 832, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1380, 2 ], "axis1": [ 832, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1383, 3 ], "axis1": [ 832, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1387, 3 ], "axis1": [ 832, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 799, 25 ], "axis1": [ 1289, 25 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 824, 2 ], "axis1": [ 1380, 2 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 826, 3 ], "axis1": [ 1383, 3 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 829, 3 ], "axis1": [ 1387, 3 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 832, 25 ], "axis1": [ 1289, 25 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 857, 2 ], "axis1": [ 1380, 2 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 859, 3 ], "axis1": [ 1383, 3 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 862, 3 ], "axis1": [ 1387, 3 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 865, 66 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1390, 33 ], "axis1": [ 865, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1390, 33 ], "axis1": [ 898, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 865, 33 ], "axis1": [ 1390, 33 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 898, 33 ], "axis1": [ 1390, 33 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 931, 66 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1423, 9 ], "axis1": [ 931, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1433, 24 ], "axis1": [ 931, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1423, 9 ], "axis1": [ 964, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1433, 24 ], "axis1": [ 964, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 931, 9 ], "axis1": [ 1423, 9 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 940, 24 ], "axis1": [ 1433, 24 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 964, 9 ], "axis1": [ 1423, 9 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 973, 24 ], "axis1": [ 1433, 24 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 997, 66 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1458, 3 ], "axis1": [ 997, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1462, 30 ], "axis1": [ 997, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1458, 3 ], "axis1": [ 1030, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1462, 30 ], "axis1": [ 1030, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 997, 3 ], "axis1": [ 1458, 3 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1000, 30 ], "axis1": [ 1462, 30 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1030, 3 ], "axis1": [ 1458, 3 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1033, 30 ], "axis1": [ 1462, 30 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1063, 66 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1493, 33 ], "axis1": [ 1063, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1493, 33 ], "axis1": [ 1096, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1063, 33 ], "axis1": [ 1493, 33 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1096, 33 ], "axis1": [ 1493, 33 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1129, 66 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1526, 9 ], "axis1": [ 1129, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1536, 22 ], "axis1": [ 1129, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1559, 2 ], "axis1": [ 1129, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1526, 9 ], "axis1": [ 1162, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1536, 22 ], "axis1": [ 1162, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1559, 2 ], "axis1": [ 1162, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1129, 9 ], "axis1": [ 1526, 9 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1138, 22 ], "axis1": [ 1536, 22 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1160, 2 ], "axis1": [ 1559, 2 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1162, 9 ], "axis1": [ 1526, 9 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1171, 22 ], "axis1": [ 1536, 22 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1193, 2 ], "axis1": [ 1559, 2 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1195, 66 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1658, 19 ], "axis1": [ 1195, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1680, 1 ], "axis1": [ 1195, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1683, 7 ], "axis1": [ 1195, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1694, 4 ], "axis1": [ 1195, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1699, 1 ], "axis1": [ 1195, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1702, 1 ], "axis1": [ 1195, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1658, 19 ], "axis1": [ 1228, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1680, 1 ], "axis1": [ 1228, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1683, 7 ], "axis1": [ 1228, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1694, 4 ], "axis1": [ 1228, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1699, 1 ], "axis1": [ 1228, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1702, 1 ], "axis1": [ 1228, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1195, 19 ], "axis1": [ 1658, 19 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1214, 1 ], "axis1": [ 1680, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1215, 7 ], "axis1": [ 1683, 7 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1222, 4 ], "axis1": [ 1694, 4 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1226, 1 ], "axis1": [ 1699, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1227, 1 ], "axis1": [ 1702, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1228, 19 ], "axis1": [ 1658, 19 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1247, 1 ], "axis1": [ 1680, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1248, 7 ], "axis1": [ 1683, 7 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1255, 4 ], "axis1": [ 1694, 4 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1259, 1 ], "axis1": [ 1699, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1260, 1 ], "axis1": [ 1702, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1261, 66 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1705, 1 ], "axis1": [ 1261, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1707, 2 ], "axis1": [ 1261, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1712, 12 ], "axis1": [ 1261, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1726, 18 ], "axis1": [ 1261, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1705, 1 ], "axis1": [ 1294, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1707, 2 ], "axis1": [ 1294, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1712, 12 ], "axis1": [ 1294, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1726, 18 ], "axis1": [ 1294, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1261, 1 ], "axis1": [ 1705, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1262, 2 ], "axis1": [ 1707, 2 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1264, 12 ], "axis1": [ 1712, 12 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1276, 18 ], "axis1": [ 1726, 18 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1294, 1 ], "axis1": [ 1705, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1295, 2 ], "axis1": [ 1707, 2 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1297, 12 ], "axis1": [ 1712, 12 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1309, 18 ], "axis1": [ 1726, 18 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1327, 66 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1744, 6 ], "axis1": [ 1327, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1763, 27 ], "axis1": [ 1327, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1744, 6 ], "axis1": [ 1360, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1763, 27 ], "axis1": [ 1360, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1327, 6 ], "axis1": [ 1744, 6 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1333, 27 ], "axis1": [ 1763, 27 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1360, 6 ], "axis1": [ 1744, 6 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1366, 27 ], "axis1": [ 1763, 27 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1393, 66 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1791, 9 ], "axis1": [ 1393, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1805, 3 ], "axis1": [ 1393, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1809, 1 ], "axis1": [ 1393, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1811, 1 ], "axis1": [ 1393, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1813, 9 ], "axis1": [ 1393, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1823, 1 ], "axis1": [ 1393, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1832, 1 ], "axis1": [ 1393, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1835, 3 ], "axis1": [ 1393, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1840, 1 ], "axis1": [ 1393, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1843, 1 ], "axis1": [ 1393, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1976, 3 ], "axis1": [ 1393, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1791, 9 ], "axis1": [ 1426, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1805, 3 ], "axis1": [ 1426, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1809, 1 ], "axis1": [ 1426, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1811, 1 ], "axis1": [ 1426, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1813, 9 ], "axis1": [ 1426, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1823, 1 ], "axis1": [ 1426, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1832, 1 ], "axis1": [ 1426, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1835, 3 ], "axis1": [ 1426, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1840, 1 ], "axis1": [ 1426, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1843, 1 ], "axis1": [ 1426, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1976, 3 ], "axis1": [ 1426, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1393, 9 ], "axis1": [ 1791, 9 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1402, 3 ], "axis1": [ 1805, 3 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1405, 1 ], "axis1": [ 1809, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1406, 1 ], "axis1": [ 1811, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1407, 9 ], "axis1": [ 1813, 9 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1416, 1 ], "axis1": [ 1823, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1417, 1 ], "axis1": [ 1832, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1418, 3 ], "axis1": [ 1835, 3 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1421, 1 ], "axis1": [ 1840, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1422, 1 ], "axis1": [ 1843, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1423, 3 ], "axis1": [ 1976, 3 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1426, 9 ], "axis1": [ 1791, 9 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1435, 3 ], "axis1": [ 1805, 3 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1438, 1 ], "axis1": [ 1809, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1439, 1 ], "axis1": [ 1811, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1440, 9 ], "axis1": [ 1813, 9 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1449, 1 ], "axis1": [ 1823, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1450, 1 ], "axis1": [ 1832, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1451, 3 ], "axis1": [ 1835, 3 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1454, 1 ], "axis1": [ 1840, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1455, 1 ], "axis1": [ 1843, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1456, 3 ], "axis1": [ 1976, 3 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1459, 66 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 1979, 33 ], "axis1": [ 1459, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1979, 33 ], "axis1": [ 1492, 33 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1459, 33 ], "axis1": [ 1979, 33 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1492, 33 ], "axis1": [ 1979, 33 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1525, 2 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 50, 1 ], "axis1": [ 1525, 1 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 50, 1 ], "axis1": [ 1526, 1 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1525, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 1526, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 317, 66 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1190, 33 ], "axis1": [ 317, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1190, 33 ], "axis1": [ 350, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 317, 33 ], "axis1": [ 1190, 33 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 350, 33 ], "axis1": [ 1190, 33 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 383, 66 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1910, 33 ], "axis1": [ 383, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1910, 33 ], "axis1": [ 416, 33 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 383, 33 ], "axis1": [ 1910, 33 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 416, 33 ], "axis1": [ 1910, 33 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 449, 2 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1844, 1 ], "axis1": [ 449, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1844, 1 ], "axis1": [ 450, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 449, 1 ], "axis1": [ 1844, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 450, 1 ], "axis1": [ 1844, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 451, 2 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1332, 1 ], "axis1": [ 451, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1332, 1 ], "axis1": [ 452, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 451, 1 ], "axis1": [ 1332, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 452, 1 ], "axis1": [ 1332, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 453, 2 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1461, 1 ], "axis1": [ 453, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1461, 1 ], "axis1": [ 454, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 453, 1 ], "axis1": [ 1461, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 454, 1 ], "axis1": [ 1461, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 455, 2 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1492, 1 ], "axis1": [ 455, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1492, 1 ], "axis1": [ 456, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 455, 1 ], "axis1": [ 1492, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 456, 1 ], "axis1": [ 1492, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 457, 2 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1369, 1 ], "axis1": [ 457, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 1369, 1 ], "axis1": [ 458, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 457, 1 ], "axis1": [ 1369, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 458, 1 ], "axis1": [ 1369, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1883, 6 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 55, 3 ], "axis1": [ 1883, 3 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 55, 3 ], "axis1": [ 1886, 3 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1883, 3 ], "axis1": [ 55, 3 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1886, 3 ], "axis1": [ 55, 3 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 204, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 244, 1 ], "axis1": [ 204, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 244, 1 ], "axis1": [ 205, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 204, 1 ], "axis1": [ 244, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 205, 1 ], "axis1": [ 244, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 206, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 246, 1 ], "axis1": [ 206, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 246, 1 ], "axis1": [ 207, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 206, 1 ], "axis1": [ 246, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 207, 1 ], "axis1": [ 246, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 208, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 959, 1 ], "axis1": [ 208, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 959, 1 ], "axis1": [ 209, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 208, 1 ], "axis1": [ 959, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 209, 1 ], "axis1": [ 959, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 210, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 988, 1 ], "axis1": [ 210, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 988, 1 ], "axis1": [ 211, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 988, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 211, 1 ], "axis1": [ 988, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 212, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 989, 1 ], "axis1": [ 212, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 989, 1 ], "axis1": [ 213, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 212, 1 ], "axis1": [ 989, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 213, 1 ], "axis1": [ 989, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 214, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 991, 1 ], "axis1": [ 214, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 991, 1 ], "axis1": [ 215, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 991, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 215, 1 ], "axis1": [ 991, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 216, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 992, 1 ], "axis1": [ 216, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 992, 1 ], "axis1": [ 217, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 216, 1 ], "axis1": [ 992, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 217, 1 ], "axis1": [ 992, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 821, 66 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 49, 3 ], "axis1": [ 821, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 53, 2 ], "axis1": [ 821, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 821, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 223, 5 ], "axis1": [ 821, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 231, 1 ], "axis1": [ 821, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 245, 2 ], "axis1": [ 821, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 254, 13 ], "axis1": [ 821, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 268, 3 ], "axis1": [ 821, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 273, 1 ], "axis1": [ 821, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 868, 1 ], "axis1": [ 821, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1150, 1 ], "axis1": [ 821, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 49, 3 ], "axis1": [ 854, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 53, 2 ], "axis1": [ 854, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 854, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 223, 5 ], "axis1": [ 854, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 231, 1 ], "axis1": [ 854, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 245, 2 ], "axis1": [ 854, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 254, 13 ], "axis1": [ 854, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 268, 3 ], "axis1": [ 854, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 273, 1 ], "axis1": [ 854, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 868, 1 ], "axis1": [ 854, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1150, 1 ], "axis1": [ 854, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 821, 3 ], "axis1": [ 49, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 824, 2 ], "axis1": [ 53, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 826, 1 ], "axis1": [ 173, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 827, 5 ], "axis1": [ 223, 5 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 832, 1 ], "axis1": [ 231, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 833, 2 ], "axis1": [ 245, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 835, 13 ], "axis1": [ 254, 13 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 848, 3 ], "axis1": [ 268, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 851, 1 ], "axis1": [ 273, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 852, 1 ], "axis1": [ 868, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 853, 1 ], "axis1": [ 1150, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 854, 3 ], "axis1": [ 49, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 857, 2 ], "axis1": [ 53, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 859, 1 ], "axis1": [ 173, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 860, 5 ], "axis1": [ 223, 5 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 865, 1 ], "axis1": [ 231, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 866, 2 ], "axis1": [ 245, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 868, 13 ], "axis1": [ 254, 13 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 881, 3 ], "axis1": [ 268, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 884, 1 ], "axis1": [ 273, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 885, 1 ], "axis1": [ 868, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 886, 1 ], "axis1": [ 1150, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 887, 66 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1131, 15 ], "axis1": [ 887, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1152, 18 ], "axis1": [ 887, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1131, 15 ], "axis1": [ 920, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1152, 18 ], "axis1": [ 920, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 887, 15 ], "axis1": [ 1131, 15 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 902, 18 ], "axis1": [ 1152, 18 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 920, 15 ], "axis1": [ 1131, 15 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 935, 18 ], "axis1": [ 1152, 18 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 953, 66 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 251, 3 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 267, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 271, 2 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 277, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 282, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 284, 2 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 287, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 292, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 307, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 312, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 317, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 321, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 323, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 439, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 514, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1027, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1065, 3 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1092, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1111, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1146, 3 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1151, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1238, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1247, 2 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1253, 1 ], "axis1": [ 953, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 251, 3 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 267, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 271, 2 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 277, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 282, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 284, 2 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 287, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 292, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 307, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 312, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 317, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 321, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 323, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 439, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 514, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1027, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1065, 3 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1092, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1111, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1146, 3 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1151, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1238, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1247, 2 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1253, 1 ], "axis1": [ 986, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 953, 3 ], "axis1": [ 251, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 956, 1 ], "axis1": [ 267, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 957, 2 ], "axis1": [ 271, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 959, 1 ], "axis1": [ 277, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 960, 1 ], "axis1": [ 282, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 961, 2 ], "axis1": [ 284, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 963, 1 ], "axis1": [ 287, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 964, 1 ], "axis1": [ 292, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 965, 1 ], "axis1": [ 307, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 966, 1 ], "axis1": [ 312, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 967, 1 ], "axis1": [ 317, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 968, 1 ], "axis1": [ 321, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 969, 1 ], "axis1": [ 323, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 970, 1 ], "axis1": [ 439, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 971, 1 ], "axis1": [ 514, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 972, 1 ], "axis1": [ 1027, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 973, 3 ], "axis1": [ 1065, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 976, 1 ], "axis1": [ 1092, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 977, 1 ], "axis1": [ 1111, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 978, 3 ], "axis1": [ 1146, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 981, 1 ], "axis1": [ 1151, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 982, 1 ], "axis1": [ 1238, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 983, 2 ], "axis1": [ 1247, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 985, 1 ], "axis1": [ 1253, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 986, 3 ], "axis1": [ 251, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 989, 1 ], "axis1": [ 267, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 990, 2 ], "axis1": [ 271, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 992, 1 ], "axis1": [ 277, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 993, 1 ], "axis1": [ 282, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 994, 2 ], "axis1": [ 284, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 996, 1 ], "axis1": [ 287, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 997, 1 ], "axis1": [ 292, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 998, 1 ], "axis1": [ 307, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 999, 1 ], "axis1": [ 312, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1000, 1 ], "axis1": [ 317, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1001, 1 ], "axis1": [ 321, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1002, 1 ], "axis1": [ 323, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1003, 1 ], "axis1": [ 439, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1004, 1 ], "axis1": [ 514, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1005, 1 ], "axis1": [ 1027, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1006, 3 ], "axis1": [ 1065, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1009, 1 ], "axis1": [ 1092, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1010, 1 ], "axis1": [ 1111, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1011, 3 ], "axis1": [ 1146, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1014, 1 ], "axis1": [ 1151, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1015, 1 ], "axis1": [ 1238, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1016, 2 ], "axis1": [ 1247, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1018, 1 ], "axis1": [ 1253, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1019, 66 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1386, 33 ], "axis1": [ 1019, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1386, 33 ], "axis1": [ 1052, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1019, 33 ], "axis1": [ 1386, 33 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1052, 33 ], "axis1": [ 1386, 33 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1085, 66 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1419, 13 ], "axis1": [ 1085, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1433, 20 ], "axis1": [ 1085, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1419, 13 ], "axis1": [ 1118, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1433, 20 ], "axis1": [ 1118, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1085, 13 ], "axis1": [ 1419, 13 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1098, 20 ], "axis1": [ 1433, 20 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1118, 13 ], "axis1": [ 1419, 13 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1131, 20 ], "axis1": [ 1433, 20 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1151, 66 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1453, 4 ], "axis1": [ 1151, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1458, 29 ], "axis1": [ 1151, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1453, 4 ], "axis1": [ 1184, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1458, 29 ], "axis1": [ 1184, 33 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1151, 4 ], "axis1": [ 1453, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1155, 29 ], "axis1": [ 1458, 29 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1184, 4 ], "axis1": [ 1453, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1188, 29 ], "axis1": [ 1458, 29 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1217, 2 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 78, 1 ], "axis1": [ 1217, 1 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 78, 1 ], "axis1": [ 1218, 1 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1217, 1 ], "axis1": [ 78, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1218, 1 ], "axis1": [ 78, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1219, 2 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1023, 1 ], "axis1": [ 1219, 1 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1023, 1 ], "axis1": [ 1220, 1 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1219, 1 ], "axis1": [ 1023, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1220, 1 ], "axis1": [ 1023, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 564, 66 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 188, 4 ], "axis1": [ 564, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 212, 1 ], "axis1": [ 564, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 228, 1 ], "axis1": [ 564, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 232, 3 ], "axis1": [ 564, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 241, 1 ], "axis1": [ 564, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 294, 1 ], "axis1": [ 564, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 297, 10 ], "axis1": [ 564, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 564, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 313, 4 ], "axis1": [ 564, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 318, 3 ], "axis1": [ 564, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 322, 1 ], "axis1": [ 564, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 188, 4 ], "axis1": [ 597, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 212, 1 ], "axis1": [ 597, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 228, 1 ], "axis1": [ 597, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 232, 3 ], "axis1": [ 597, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 241, 1 ], "axis1": [ 597, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 294, 1 ], "axis1": [ 597, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 297, 10 ], "axis1": [ 597, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 597, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 313, 4 ], "axis1": [ 597, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 318, 3 ], "axis1": [ 597, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 322, 1 ], "axis1": [ 597, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 564, 4 ], "axis1": [ 188, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 568, 1 ], "axis1": [ 212, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 569, 1 ], "axis1": [ 228, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 570, 3 ], "axis1": [ 232, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 573, 1 ], "axis1": [ 241, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 574, 1 ], "axis1": [ 294, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 575, 10 ], "axis1": [ 297, 10 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 585, 4 ], "axis1": [ 308, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 589, 4 ], "axis1": [ 313, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 593, 3 ], "axis1": [ 318, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 596, 1 ], "axis1": [ 322, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 597, 4 ], "axis1": [ 188, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 601, 1 ], "axis1": [ 212, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 602, 1 ], "axis1": [ 228, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 603, 3 ], "axis1": [ 232, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 606, 1 ], "axis1": [ 241, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 607, 1 ], "axis1": [ 294, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 608, 10 ], "axis1": [ 297, 10 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 618, 4 ], "axis1": [ 308, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 622, 4 ], "axis1": [ 313, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 626, 3 ], "axis1": [ 318, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 629, 1 ], "axis1": [ 322, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 630, 66 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 676, 8 ], "axis1": [ 630, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 869, 8 ], "axis1": [ 630, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 878, 1 ], "axis1": [ 630, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 883, 5 ], "axis1": [ 630, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 930, 1 ], "axis1": [ 630, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 965, 10 ], "axis1": [ 630, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 676, 8 ], "axis1": [ 663, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 869, 8 ], "axis1": [ 663, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 878, 1 ], "axis1": [ 663, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 883, 5 ], "axis1": [ 663, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 930, 1 ], "axis1": [ 663, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 965, 10 ], "axis1": [ 663, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 630, 8 ], "axis1": [ 676, 8 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 638, 8 ], "axis1": [ 869, 8 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 646, 1 ], "axis1": [ 878, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 647, 5 ], "axis1": [ 883, 5 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 652, 1 ], "axis1": [ 930, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 653, 10 ], "axis1": [ 965, 10 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 663, 8 ], "axis1": [ 676, 8 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 671, 8 ], "axis1": [ 869, 8 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 679, 1 ], "axis1": [ 878, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 680, 5 ], "axis1": [ 883, 5 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 685, 1 ], "axis1": [ 930, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 686, 10 ], "axis1": [ 965, 10 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 696, 66 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 247, 4 ], "axis1": [ 696, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1060, 1 ], "axis1": [ 696, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1062, 1 ], "axis1": [ 696, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1064, 1 ], "axis1": [ 696, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1226, 12 ], "axis1": [ 696, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1239, 4 ], "axis1": [ 696, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1244, 3 ], "axis1": [ 696, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1249, 4 ], "axis1": [ 696, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1254, 3 ], "axis1": [ 696, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 247, 4 ], "axis1": [ 729, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1060, 1 ], "axis1": [ 729, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1062, 1 ], "axis1": [ 729, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1064, 1 ], "axis1": [ 729, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1226, 12 ], "axis1": [ 729, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1239, 4 ], "axis1": [ 729, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1244, 3 ], "axis1": [ 729, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1249, 4 ], "axis1": [ 729, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1254, 3 ], "axis1": [ 729, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 696, 4 ], "axis1": [ 247, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 700, 1 ], "axis1": [ 1060, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 701, 1 ], "axis1": [ 1062, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 702, 1 ], "axis1": [ 1064, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 703, 12 ], "axis1": [ 1226, 12 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 715, 4 ], "axis1": [ 1239, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 719, 3 ], "axis1": [ 1244, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 722, 4 ], "axis1": [ 1249, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 726, 3 ], "axis1": [ 1254, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 729, 4 ], "axis1": [ 247, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 733, 1 ], "axis1": [ 1060, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 734, 1 ], "axis1": [ 1062, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 735, 1 ], "axis1": [ 1064, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 736, 12 ], "axis1": [ 1226, 12 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 748, 4 ], "axis1": [ 1239, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 752, 3 ], "axis1": [ 1244, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 755, 4 ], "axis1": [ 1249, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 759, 3 ], "axis1": [ 1254, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 762, 66 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 975, 7 ], "axis1": [ 762, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 984, 2 ], "axis1": [ 762, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1006, 4 ], "axis1": [ 762, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1030, 1 ], "axis1": [ 762, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1084, 3 ], "axis1": [ 762, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1088, 1 ], "axis1": [ 762, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1243, 1 ], "axis1": [ 762, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1487, 14 ], "axis1": [ 762, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 975, 7 ], "axis1": [ 795, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 984, 2 ], "axis1": [ 795, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1006, 4 ], "axis1": [ 795, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1030, 1 ], "axis1": [ 795, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1084, 3 ], "axis1": [ 795, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1088, 1 ], "axis1": [ 795, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1243, 1 ], "axis1": [ 795, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1487, 14 ], "axis1": [ 795, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 762, 7 ], "axis1": [ 975, 7 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 769, 2 ], "axis1": [ 984, 2 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 771, 4 ], "axis1": [ 1006, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 775, 1 ], "axis1": [ 1030, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 776, 3 ], "axis1": [ 1084, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 779, 1 ], "axis1": [ 1088, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 780, 1 ], "axis1": [ 1243, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 781, 14 ], "axis1": [ 1487, 14 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 795, 7 ], "axis1": [ 975, 7 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 802, 2 ], "axis1": [ 984, 2 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 804, 4 ], "axis1": [ 1006, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 808, 1 ], "axis1": [ 1030, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 809, 3 ], "axis1": [ 1084, 3 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 812, 1 ], "axis1": [ 1088, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 813, 1 ], "axis1": [ 1243, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 814, 14 ], "axis1": [ 1487, 14 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 828, 66 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1501, 19 ], "axis1": [ 828, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1554, 4 ], "axis1": [ 828, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1559, 10 ], "axis1": [ 828, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1501, 19 ], "axis1": [ 861, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1554, 4 ], "axis1": [ 861, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1559, 10 ], "axis1": [ 861, 33 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 828, 19 ], "axis1": [ 1501, 19 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 847, 4 ], "axis1": [ 1554, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 851, 10 ], "axis1": [ 1559, 10 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 861, 19 ], "axis1": [ 1501, 19 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 880, 4 ], "axis1": [ 1554, 4 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 884, 10 ], "axis1": [ 1559, 10 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 894, 2 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1638, 1 ], "axis1": [ 894, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1638, 1 ], "axis1": [ 895, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 894, 1 ], "axis1": [ 1638, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 895, 1 ], "axis1": [ 1638, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 2 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1679, 1 ], "axis1": [ 896, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1679, 1 ], "axis1": [ 897, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 896, 1 ], "axis1": [ 1679, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 897, 1 ], "axis1": [ 1679, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 898, 2 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 295, 1 ], "axis1": [ 898, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 295, 1 ], "axis1": [ 899, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 898, 1 ], "axis1": [ 295, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 899, 1 ], "axis1": [ 295, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 900, 2 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1535, 1 ], "axis1": [ 900, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1535, 1 ], "axis1": [ 901, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 900, 1 ], "axis1": [ 1535, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 901, 1 ], "axis1": [ 1535, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 902, 2 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 1457, 1 ], "axis1": [ 902, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1457, 1 ], "axis1": [ 903, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 902, 1 ], "axis1": [ 1457, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 903, 1 ], "axis1": [ 1457, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 265, 66 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1613, 25 ], "axis1": [ 265, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1639, 4 ], "axis1": [ 265, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1644, 1 ], "axis1": [ 265, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1827, 1 ], "axis1": [ 265, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1833, 2 ], "axis1": [ 265, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1613, 25 ], "axis1": [ 298, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1639, 4 ], "axis1": [ 298, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1644, 1 ], "axis1": [ 298, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1827, 1 ], "axis1": [ 298, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1833, 2 ], "axis1": [ 298, 33 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 265, 25 ], "axis1": [ 1613, 25 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 290, 4 ], "axis1": [ 1639, 4 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 294, 1 ], "axis1": [ 1644, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 295, 1 ], "axis1": [ 1827, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 296, 2 ], "axis1": [ 1833, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 298, 25 ], "axis1": [ 1613, 25 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 323, 4 ], "axis1": [ 1639, 4 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 327, 1 ], "axis1": [ 1644, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 328, 1 ], "axis1": [ 1827, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 329, 2 ], "axis1": [ 1833, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 331, 2 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1046, 1 ], "axis1": [ 331, 1 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1046, 1 ], "axis1": [ 332, 1 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 331, 1 ], "axis1": [ 1046, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 332, 1 ], "axis1": [ 1046, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 333, 2 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1061, 1 ], "axis1": [ 333, 1 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1061, 1 ], "axis1": [ 334, 1 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 1061, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 334, 1 ], "axis1": [ 1061, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 335, 2 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 1432, 1 ], "axis1": [ 335, 1 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1432, 1 ], "axis1": [ 336, 1 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 335, 1 ], "axis1": [ 1432, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 336, 1 ], "axis1": [ 1432, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 67, 66 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 1854, 1 ], "axis1": [ 67, 33 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1856, 1 ], "axis1": [ 67, 33 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1859, 17 ], "axis1": [ 67, 33 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1877, 1 ], "axis1": [ 67, 33 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1889, 13 ], "axis1": [ 67, 33 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1854, 1 ], "axis1": [ 100, 33 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1856, 1 ], "axis1": [ 100, 33 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1859, 17 ], "axis1": [ 100, 33 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1877, 1 ], "axis1": [ 100, 33 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1889, 13 ], "axis1": [ 100, 33 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 1854, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 68, 1 ], "axis1": [ 1856, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 69, 17 ], "axis1": [ 1859, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 86, 1 ], "axis1": [ 1877, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 87, 13 ], "axis1": [ 1889, 13 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 100, 1 ], "axis1": [ 1854, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 101, 1 ], "axis1": [ 1856, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 102, 17 ], "axis1": [ 1859, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 119, 1 ], "axis1": [ 1877, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 120, 13 ], "axis1": [ 1889, 13 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 133, 2 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 1790, 1 ], "axis1": [ 133, 1 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1790, 1 ], "axis1": [ 134, 1 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 133, 1 ], "axis1": [ 1790, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 134, 1 ], "axis1": [ 1790, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" } ] } }, "reference_fixtures": { "included": [], "reason": "No input corpus was supplied to compile_hf_bundle.", "scope": "Input-specific activations and intervention results." }, "residual_stream": { "constant_one_column": 0, "constant_one_semantics": "Holds exactly 1.0 at every position. Read as the \u0394=0 self-match query/key source by every attention operation except compute_attn (their query/key source columns) and, under bias=False, by MLP bias folds (their W_in/W_up rows in physical_layout.placements). Patching it collapses every self-match head.", "rms_norm_reserved_columns": [ 2046, 2047 ], "state_semantics": "input and post-layer residual streams; the compiler has no stable observation boundary between attention and MLP", "states": [ { "key": "input", "nodes": { "i:0": [ [ 0, 1 ] ], "l:8": [ [ 1, 33 ] ] } }, { "key": "L0.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1002": [ [ 531, 33 ] ], "l:1005": [ [ 762, 33 ] ], "l:1013": [ [ 149, 1 ] ], "l:102": [ [ 81, 1 ] ], "l:1026": [ [ 564, 33 ] ], "l:1037": [ [ 150, 1 ] ], "l:1050": [ [ 597, 33 ] ], "l:1053": [ [ 795, 33 ] ], "l:1056": [ [ 366, 33 ] ], "l:106": [ [ 66, 1 ] ], "l:109": [ [ 61, 1 ] ], "l:110": [ [ 62, 1 ] ], "l:112": [ [ 42, 1 ] ], "l:113": [ [ 63, 1 ] ], "l:121": [ [ 43, 1 ] ], "l:125": [ [ 67, 1 ] ], "l:126": [ [ 68, 1 ] ], "l:128": [ [ 69, 1 ] ], "l:134": [ [ 84, 1 ] ], "l:137": [ [ 73, 4 ] ], "l:140": [ [ 85, 33 ] ], "l:155": [ [ 82, 1 ] ], "l:162": [ [ 64, 1 ] ], "l:163": [ [ 65, 1 ] ], "l:179": [ [ 118, 1 ] ], "l:182": [ [ 77, 4 ] ], "l:191": [ [ 55, 3 ] ], "l:195": [ [ 119, 1 ] ], "l:20": [ [ 83, 1 ] ], "l:23": [ [ 70, 1 ] ], "l:24": [ [ 71, 1 ] ], "l:242": [ [ 59, 1 ] ], "l:26": [ [ 44, 1 ] ], "l:260": [ [ 60, 1 ] ], "l:27": [ [ 72, 1 ] ], "l:310": [ [ 45, 1 ] ], "l:311": [ [ 34, 1 ] ], "l:340": [ [ 160, 1 ] ], "l:351": [ [ 120, 1 ] ], "l:358": [ [ 58, 1 ] ], "l:37": [ [ 161, 1 ] ], "l:381": [ [ 153, 1 ] ], "l:392": [ [ 121, 1 ] ], "l:4": [ [ 828, 1 ] ], "l:418": [ [ 154, 1 ] ], "l:429": [ [ 122, 1 ] ], "l:436": [ [ 151, 1 ] ], "l:44": [ [ 38, 1 ] ], "l:443": [ [ 155, 1 ] ], "l:46": [ [ 39, 1 ] ], "l:465": [ [ 123, 1 ] ], "l:472": [ [ 152, 1 ] ], "l:479": [ [ 156, 1 ] ], "l:48": [ [ 40, 1 ] ], "l:5": [ [ 829, 1 ] ], "l:503": [ [ 128, 1 ] ], "l:513": [ [ 157, 1 ] ], "l:546": [ [ 158, 1 ] ], "l:559": [ [ 49, 1 ] ], "l:56": [ [ 162, 1 ] ], "l:566": [ [ 133, 1 ] ], "l:571": [ [ 35, 1 ] ], "l:575": [ [ 46, 1 ] ], "l:578": [ [ 135, 1 ] ], "l:585": [ [ 159, 1 ] ], "l:598": [ [ 50, 1 ] ], "l:605": [ [ 134, 1 ] ], "l:610": [ [ 36, 1 ] ], "l:614": [ [ 47, 1 ] ], "l:617": [ [ 136, 1 ] ], "l:63": [ [ 163, 1 ] ], "l:632": [ [ 168, 33 ] ], "l:635": [ [ 267, 33 ] ], "l:65": [ [ 164, 1 ] ], "l:66": [ [ 234, 33 ] ], "l:660": [ [ 145, 1 ] ], "l:673": [ [ 129, 1 ] ], "l:681": [ [ 124, 1 ] ], "l:682": [ [ 137, 1 ] ], "l:694": [ [ 51, 1 ] ], "l:7": [ [ 41, 1 ] ], "l:701": [ [ 140, 1 ] ], "l:72": [ [ 165, 1 ] ], "l:730": [ [ 125, 1 ] ], "l:731": [ [ 138, 1 ] ], "l:735": [ [ 130, 1 ] ], "l:74": [ [ 166, 1 ] ], "l:743": [ [ 52, 1 ] ], "l:75": [ [ 300, 33 ] ], "l:750": [ [ 141, 1 ] ], "l:779": [ [ 126, 1 ] ], "l:784": [ [ 131, 1 ] ], "l:791": [ [ 53, 1 ] ], "l:798": [ [ 142, 1 ] ], "l:8": [ [ 1, 33 ] ], "l:814": [ [ 146, 1 ] ], "l:827": [ [ 127, 1 ] ], "l:828": [ [ 139, 1 ] ], "l:832": [ [ 132, 1 ] ], "l:839": [ [ 54, 1 ] ], "l:846": [ [ 143, 1 ] ], "l:857": [ [ 201, 33 ] ], "l:865": [ [ 37, 1 ] ], "l:869": [ [ 48, 1 ] ], "l:872": [ [ 144, 1 ] ], "l:883": [ [ 399, 33 ] ], "l:886": [ [ 630, 33 ] ], "l:889": [ [ 333, 33 ] ], "l:9": [ [ 830, 1 ] ], "l:903": [ [ 663, 33 ] ], "l:908": [ [ 167, 1 ] ], "l:914": [ [ 147, 1 ] ], "l:930": [ [ 432, 33 ] ], "l:933": [ [ 696, 33 ] ], "l:954": [ [ 465, 33 ] ], "l:978": [ [ 498, 33 ] ], "l:981": [ [ 729, 33 ] ], "l:989": [ [ 148, 1 ] ] } }, { "key": "L1.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1002": [ [ 531, 33 ] ], "l:1005": [ [ 762, 33 ] ], "l:1013": [ [ 149, 1 ] ], "l:102": [ [ 81, 1 ] ], "l:1026": [ [ 564, 33 ] ], "l:1037": [ [ 150, 1 ] ], "l:1050": [ [ 597, 33 ] ], "l:1053": [ [ 795, 33 ] ], "l:1056": [ [ 366, 33 ] ], "l:106": [ [ 66, 1 ] ], "l:107": [ [ 831, 1 ] ], "l:114": [ [ 1831, 1 ] ], "l:121": [ [ 43, 1 ] ], "l:125": [ [ 67, 1 ] ], "l:126": [ [ 68, 1 ] ], "l:128": [ [ 69, 1 ] ], "l:134": [ [ 84, 1 ] ], "l:135": [ [ 1800, 4 ] ], "l:137": [ [ 73, 4 ] ], "l:140": [ [ 85, 33 ] ], "l:155": [ [ 82, 1 ] ], "l:162": [ [ 64, 1 ] ], "l:163": [ [ 65, 1 ] ], "l:179": [ [ 118, 1 ] ], "l:182": [ [ 77, 4 ] ], "l:191": [ [ 55, 3 ] ], "l:195": [ [ 119, 1 ] ], "l:2": [ [ 1790, 1 ] ], "l:20": [ [ 83, 1 ] ], "l:21": [ [ 832, 1 ] ], "l:242": [ [ 59, 1 ] ], "l:26": [ [ 44, 1 ] ], "l:260": [ [ 60, 1 ] ], "l:28": [ [ 1832, 1 ] ], "l:310": [ [ 45, 1 ] ], "l:340": [ [ 160, 1 ] ], "l:349": [ [ 1813, 1 ] ], "l:350": [ [ 833, 33 ] ], "l:357": [ [ 1833, 1 ] ], "l:358": [ [ 58, 1 ] ], "l:37": [ [ 161, 1 ] ], "l:379": [ [ 1824, 1 ] ], "l:380": [ [ 1559, 33 ] ], "l:390": [ [ 1814, 1 ] ], "l:391": [ [ 866, 33 ] ], "l:398": [ [ 1834, 1 ] ], "l:416": [ [ 1825, 1 ] ], "l:417": [ [ 1592, 33 ] ], "l:427": [ [ 1815, 1 ] ], "l:428": [ [ 899, 33 ] ], "l:43": [ [ 1804, 1 ] ], "l:434": [ [ 1822, 1 ] ], "l:435": [ [ 1493, 33 ] ], "l:44": [ [ 38, 1 ] ], "l:441": [ [ 1826, 1 ] ], "l:442": [ [ 1625, 33 ] ], "l:45": [ [ 1805, 1 ] ], "l:46": [ [ 39, 1 ] ], "l:463": [ [ 1816, 1 ] ], "l:464": [ [ 932, 33 ] ], "l:47": [ [ 1806, 1 ] ], "l:470": [ [ 1823, 1 ] ], "l:471": [ [ 1526, 33 ] ], "l:477": [ [ 1827, 1 ] ], "l:478": [ [ 1658, 33 ] ], "l:48": [ [ 40, 1 ] ], "l:493": [ [ 1843, 1 ] ], "l:501": [ [ 1818, 1 ] ], "l:502": [ [ 1064, 33 ] ], "l:511": [ [ 1828, 1 ] ], "l:512": [ [ 1691, 33 ] ], "l:52": [ [ 1835, 1 ] ], "l:53": [ [ 1836, 1 ] ], "l:544": [ [ 1829, 1 ] ], "l:545": [ [ 1724, 33 ] ], "l:55": [ [ 1837, 1 ] ], "l:559": [ [ 49, 1 ] ], "l:56": [ [ 162, 1 ] ], "l:563": [ [ 1807, 1 ] ], "l:564": [ [ 1791, 1 ] ], "l:565": [ [ 1196, 33 ] ], "l:571": [ [ 35, 1 ] ], "l:575": [ [ 46, 1 ] ], "l:576": [ [ 1793, 1 ] ], "l:577": [ [ 1262, 33 ] ], "l:583": [ [ 1830, 1 ] ], "l:584": [ [ 1757, 33 ] ], "l:598": [ [ 50, 1 ] ], "l:602": [ [ 1808, 1 ] ], "l:603": [ [ 1792, 1 ] ], "l:604": [ [ 1229, 33 ] ], "l:610": [ [ 36, 1 ] ], "l:614": [ [ 47, 1 ] ], "l:615": [ [ 1794, 1 ] ], "l:616": [ [ 1295, 33 ] ], "l:63": [ [ 163, 1 ] ], "l:632": [ [ 168, 33 ] ], "l:635": [ [ 267, 33 ] ], "l:641": [ [ 1844, 1 ] ], "l:65": [ [ 164, 1 ] ], "l:66": [ [ 234, 33 ] ], "l:660": [ [ 145, 1 ] ], "l:671": [ [ 1819, 1 ] ], "l:672": [ [ 1097, 33 ] ], "l:681": [ [ 124, 1 ] ], "l:682": [ [ 137, 1 ] ], "l:694": [ [ 51, 1 ] ], "l:698": [ [ 1809, 1 ] ], "l:699": [ [ 1795, 1 ] ], "l:7": [ [ 41, 1 ] ], "l:700": [ [ 1328, 33 ] ], "l:717": [ [ 1839, 1 ] ], "l:72": [ [ 165, 1 ] ], "l:729": [ [ 965, 33 ] ], "l:731": [ [ 138, 1 ] ], "l:735": [ [ 130, 1 ] ], "l:74": [ [ 166, 1 ] ], "l:743": [ [ 52, 1 ] ], "l:747": [ [ 1810, 1 ] ], "l:748": [ [ 1796, 1 ] ], "l:749": [ [ 1361, 33 ] ], "l:75": [ [ 300, 33 ] ], "l:766": [ [ 1840, 1 ] ], "l:778": [ [ 998, 33 ] ], "l:780": [ [ 1838, 1 ] ], "l:782": [ [ 1820, 1 ] ], "l:783": [ [ 1130, 33 ] ], "l:791": [ [ 53, 1 ] ], "l:795": [ [ 1811, 1 ] ], "l:796": [ [ 1797, 1 ] ], "l:797": [ [ 1394, 33 ] ], "l:8": [ [ 1, 33 ] ], "l:814": [ [ 146, 1 ] ], "l:825": [ [ 1817, 1 ] ], "l:826": [ [ 1031, 33 ] ], "l:828": [ [ 139, 1 ] ], "l:830": [ [ 1821, 1 ] ], "l:831": [ [ 1163, 33 ] ], "l:839": [ [ 54, 1 ] ], "l:843": [ [ 1812, 1 ] ], "l:844": [ [ 1798, 1 ] ], "l:845": [ [ 1427, 33 ] ], "l:857": [ [ 201, 33 ] ], "l:865": [ [ 37, 1 ] ], "l:869": [ [ 48, 1 ] ], "l:870": [ [ 1799, 1 ] ], "l:871": [ [ 1460, 33 ] ], "l:883": [ [ 399, 33 ] ], "l:886": [ [ 630, 33 ] ], "l:889": [ [ 333, 33 ] ], "l:903": [ [ 663, 33 ] ], "l:908": [ [ 167, 1 ] ], "l:914": [ [ 147, 1 ] ], "l:930": [ [ 432, 33 ] ], "l:933": [ [ 696, 33 ] ], "l:941": [ [ 1841, 1 ] ], "l:954": [ [ 465, 33 ] ], "l:965": [ [ 1842, 1 ] ], "l:978": [ [ 498, 33 ] ], "l:981": [ [ 729, 33 ] ], "l:989": [ [ 148, 1 ] ] } }, { "key": "L2.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1002": [ [ 531, 33 ] ], "l:1005": [ [ 762, 33 ] ], "l:1011": [ [ 1015, 1 ] ], "l:1012": [ [ 156, 5 ], [ 828, 20 ], [ 1660, 2 ], [ 1784, 6 ] ], "l:102": [ [ 81, 1 ] ], "l:1026": [ [ 564, 33 ] ], "l:1035": [ [ 1016, 1 ] ], "l:1036": [ [ 34, 1 ], [ 848, 13 ], [ 862, 19 ] ], "l:104": [ [ 985, 1 ] ], "l:105": [ [ 1663, 1 ] ], "l:1050": [ [ 597, 33 ] ], "l:1053": [ [ 795, 33 ] ], "l:1056": [ [ 366, 33 ] ], "l:121": [ [ 43, 1 ] ], "l:125": [ [ 67, 1 ] ], "l:126": [ [ 68, 1 ] ], "l:128": [ [ 69, 1 ] ], "l:134": [ [ 84, 1 ] ], "l:135": [ [ 1800, 4 ] ], "l:140": [ [ 85, 33 ] ], "l:155": [ [ 82, 1 ] ], "l:162": [ [ 64, 1 ] ], "l:163": [ [ 65, 1 ] ], "l:173": [ [ 986, 1 ] ], "l:179": [ [ 118, 1 ] ], "l:18": [ [ 987, 1 ] ], "l:182": [ [ 77, 4 ] ], "l:19": [ [ 1635, 1 ] ], "l:191": [ [ 55, 3 ] ], "l:195": [ [ 119, 1 ] ], "l:2": [ [ 1790, 1 ] ], "l:242": [ [ 59, 1 ] ], "l:260": [ [ 60, 1 ] ], "l:310": [ [ 45, 1 ] ], "l:338": [ [ 1017, 1 ] ], "l:339": [ [ 72, 1 ], [ 881, 8 ], [ 890, 24 ] ], "l:348": [ [ 988, 1 ] ], "l:357": [ [ 1833, 1 ] ], "l:37": [ [ 161, 1 ] ], "l:379": [ [ 1824, 1 ] ], "l:389": [ [ 989, 1 ] ], "l:398": [ [ 1834, 1 ] ], "l:41": [ [ 983, 1 ] ], "l:416": [ [ 1825, 1 ] ], "l:426": [ [ 990, 1 ] ], "l:43": [ [ 1804, 1 ] ], "l:434": [ [ 1822, 1 ] ], "l:44": [ [ 38, 1 ] ], "l:441": [ [ 1826, 1 ] ], "l:46": [ [ 39, 1 ] ], "l:462": [ [ 991, 1 ] ], "l:469": [ [ 982, 1 ] ], "l:47": [ [ 1806, 1 ] ], "l:477": [ [ 1827, 1 ] ], "l:48": [ [ 40, 1 ] ], "l:49": [ [ 984, 1 ] ], "l:491": [ [ 1009, 1 ] ], "l:492": [ [ 914, 33 ] ], "l:50": [ [ 1662, 1 ] ], "l:500": [ [ 992, 1 ] ], "l:501": [ [ 1818, 1 ] ], "l:511": [ [ 1828, 1 ] ], "l:544": [ [ 1829, 1 ] ], "l:559": [ [ 49, 1 ] ], "l:56": [ [ 162, 1 ] ], "l:561": [ [ 999, 1 ] ], "l:564": [ [ 1791, 1 ] ], "l:571": [ [ 35, 1 ] ], "l:573": [ [ 1001, 1 ] ], "l:575": [ [ 46, 1 ] ], "l:583": [ [ 1830, 1 ] ], "l:598": [ [ 50, 1 ] ], "l:600": [ [ 1000, 1 ] ], "l:602": [ [ 1808, 1 ] ], "l:603": [ [ 1792, 1 ] ], "l:610": [ [ 36, 1 ] ], "l:612": [ [ 1002, 1 ] ], "l:615": [ [ 1794, 1 ] ], "l:63": [ [ 163, 1 ] ], "l:632": [ [ 168, 33 ] ], "l:635": [ [ 267, 33 ] ], "l:639": [ [ 1010, 1 ] ], "l:640": [ [ 947, 33 ] ], "l:65": [ [ 164, 1 ] ], "l:66": [ [ 234, 33 ] ], "l:660": [ [ 145, 1 ] ], "l:670": [ [ 1008, 1 ] ], "l:671": [ [ 1819, 1 ] ], "l:680": [ [ 1636, 3 ], [ 1640, 1 ], [ 1643, 1 ], [ 1664, 17 ], [ 1688, 6 ], [ 1695, 2 ], [ 1702, 2 ], [ 1706, 1 ] ], "l:682": [ [ 137, 1 ] ], "l:683": [ [ 998, 1 ] ], "l:694": [ [ 51, 1 ] ], "l:696": [ [ 1003, 1 ] ], "l:699": [ [ 1795, 1 ] ], "l:7": [ [ 41, 1 ] ], "l:717": [ [ 1839, 1 ] ], "l:72": [ [ 165, 1 ] ], "l:726": [ [ 993, 1 ] ], "l:728": [ [ 1823, 1 ] ], "l:731": [ [ 138, 1 ] ], "l:733": [ [ 1011, 1 ] ], "l:734": [ [ 83, 1 ], [ 1644, 5 ], [ 1694, 1 ], [ 1697, 1 ], [ 1708, 25 ] ], "l:74": [ [ 166, 1 ] ], "l:743": [ [ 52, 1 ] ], "l:745": [ [ 1004, 1 ] ], "l:747": [ [ 1810, 1 ] ], "l:748": [ [ 1796, 1 ] ], "l:75": [ [ 300, 33 ] ], "l:764": [ [ 1012, 1 ] ], "l:765": [ [ 120, 5 ], [ 1642, 1 ], [ 1649, 5 ], [ 1698, 1 ], [ 1707, 1 ], [ 1733, 20 ] ], "l:775": [ [ 994, 1 ] ], "l:777": [ [ 981, 1 ] ], "l:780": [ [ 1838, 1 ] ], "l:781": [ [ 997, 1 ] ], "l:791": [ [ 53, 1 ] ], "l:793": [ [ 1005, 1 ] ], "l:796": [ [ 1797, 1 ] ], "l:8": [ [ 1, 33 ] ], "l:814": [ [ 146, 1 ] ], "l:823": [ [ 995, 1 ] ], "l:825": [ [ 1817, 1 ] ], "l:829": [ [ 996, 1 ] ], "l:839": [ [ 54, 1 ] ], "l:841": [ [ 1006, 1 ] ], "l:843": [ [ 1812, 1 ] ], "l:857": [ [ 201, 33 ] ], "l:865": [ [ 37, 1 ] ], "l:867": [ [ 1007, 1 ] ], "l:883": [ [ 399, 33 ] ], "l:886": [ [ 630, 33 ] ], "l:889": [ [ 333, 33 ] ], "l:903": [ [ 663, 33 ] ], "l:908": [ [ 167, 1 ] ], "l:914": [ [ 147, 1 ] ], "l:930": [ [ 432, 33 ] ], "l:933": [ [ 696, 33 ] ], "l:941": [ [ 1841, 1 ] ], "l:954": [ [ 465, 33 ] ], "l:963": [ [ 1013, 1 ] ], "l:964": [ [ 66, 1 ], [ 70, 1 ], [ 125, 12 ], [ 1654, 3 ], [ 1753, 16 ] ], "l:978": [ [ 498, 33 ] ], "l:981": [ [ 729, 33 ] ], "l:987": [ [ 73, 1 ] ], "l:988": [ [ 71, 1 ], [ 140, 5 ], [ 148, 8 ], [ 1657, 3 ], [ 1699, 1 ], [ 1769, 15 ] ] } }, { "key": "L3.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:100": [ [ 1843, 1 ] ], "l:1002": [ [ 531, 33 ] ], "l:1005": [ [ 762, 33 ] ], "l:1010": [ [ 884, 1 ] ], "l:1011": [ [ 1015, 1 ] ], "l:1015": [ [ 853, 2 ], [ 856, 3 ], [ 1798, 1 ] ], "l:102": [ [ 81, 1 ] ], "l:1020": [ [ 1064, 1 ] ], "l:1021": [ [ 1065, 1 ] ], "l:1026": [ [ 564, 33 ] ], "l:103": [ [ 1068, 1 ] ], "l:1034": [ [ 883, 1 ] ], "l:1035": [ [ 1016, 1 ] ], "l:1039": [ [ 847, 3 ], [ 851, 1 ], [ 1662, 1 ], [ 1797, 1 ] ], "l:104": [ [ 985, 1 ] ], "l:1044": [ [ 1840, 1 ] ], "l:1045": [ [ 1067, 1 ] ], "l:1050": [ [ 597, 33 ] ], "l:1053": [ [ 795, 33 ] ], "l:1056": [ [ 366, 33 ] ], "l:121": [ [ 43, 1 ] ], "l:125": [ [ 67, 1 ] ], "l:126": [ [ 68, 1 ] ], "l:128": [ [ 69, 1 ] ], "l:134": [ [ 84, 1 ] ], "l:135": [ [ 1800, 4 ] ], "l:140": [ [ 85, 33 ] ], "l:155": [ [ 82, 1 ] ], "l:16": [ [ 1004, 1 ] ], "l:162": [ [ 64, 1 ] ], "l:163": [ [ 65, 1 ] ], "l:165": [ [ 1079, 1 ] ], "l:17": [ [ 836, 1 ] ], "l:173": [ [ 986, 1 ] ], "l:179": [ [ 118, 1 ] ], "l:182": [ [ 77, 4 ] ], "l:191": [ [ 55, 3 ] ], "l:195": [ [ 119, 1 ] ], "l:2": [ [ 1790, 1 ] ], "l:215": [ [ 1018, 1 ] ], "l:216": [ [ 1019, 1 ] ], "l:242": [ [ 59, 1 ] ], "l:260": [ [ 60, 1 ] ], "l:265": [ [ 1020, 1 ] ], "l:266": [ [ 1831, 1 ] ], "l:29": [ [ 1005, 1 ] ], "l:297": [ [ 1022, 1 ] ], "l:298": [ [ 1026, 1 ] ], "l:310": [ [ 45, 1 ] ], "l:331": [ [ 1024, 1 ] ], "l:332": [ [ 1025, 1 ] ], "l:337": [ [ 954, 1 ], [ 956, 4 ], [ 961, 4 ], [ 966, 4 ], [ 971, 9 ], [ 981, 1 ], [ 987, 1 ], [ 999, 4 ], [ 1818, 4 ], [ 1823, 1 ] ], "l:338": [ [ 1017, 1 ] ], "l:348": [ [ 988, 1 ] ], "l:357": [ [ 1833, 1 ] ], "l:36": [ [ 1082, 1 ] ], "l:37": [ [ 161, 1 ] ], "l:371": [ [ 1832, 1 ] ], "l:372": [ [ 1027, 1 ] ], "l:379": [ [ 1824, 1 ] ], "l:389": [ [ 989, 1 ] ], "l:39": [ [ 1796, 1 ] ], "l:398": [ [ 1834, 1 ] ], "l:408": [ [ 1028, 1 ] ], "l:409": [ [ 1029, 1 ] ], "l:416": [ [ 1825, 1 ] ], "l:426": [ [ 990, 1 ] ], "l:434": [ [ 1822, 1 ] ], "l:44": [ [ 38, 1 ] ], "l:441": [ [ 1826, 1 ] ], "l:448": [ [ 1030, 1 ] ], "l:449": [ [ 1031, 1 ] ], "l:46": [ [ 39, 1 ] ], "l:462": [ [ 991, 1 ] ], "l:469": [ [ 982, 1 ] ], "l:477": [ [ 1827, 1 ] ], "l:48": [ [ 40, 1 ] ], "l:484": [ [ 1032, 1 ] ], "l:485": [ [ 1033, 1 ] ], "l:490": [ [ 921, 9 ], [ 931, 4 ], [ 936, 4 ], [ 941, 4 ], [ 946, 4 ], [ 951, 3 ], [ 1813, 5 ] ], "l:491": [ [ 1009, 1 ] ], "l:500": [ [ 992, 1 ] ], "l:511": [ [ 1828, 1 ] ], "l:518": [ [ 1034, 1 ] ], "l:519": [ [ 1035, 1 ] ], "l:544": [ [ 1829, 1 ] ], "l:551": [ [ 1036, 1 ] ], "l:552": [ [ 1037, 1 ] ], "l:558": [ [ 49, 1 ] ], "l:56": [ [ 162, 1 ] ], "l:560": [ [ 837, 1 ] ], "l:57": [ [ 44, 1 ], [ 1085, 26 ], [ 1112, 6 ] ], "l:570": [ [ 35, 1 ] ], "l:572": [ [ 839, 1 ] ], "l:583": [ [ 1830, 1 ] ], "l:590": [ [ 1038, 1 ] ], "l:591": [ [ 1039, 1 ] ], "l:597": [ [ 50, 1 ] ], "l:599": [ [ 838, 1 ] ], "l:609": [ [ 36, 1 ] ], "l:611": [ [ 1795, 1 ] ], "l:62": [ [ 1083, 1 ] ], "l:626": [ [ 1040, 1 ] ], "l:627": [ [ 1835, 1 ] ], "l:63": [ [ 163, 1 ] ], "l:632": [ [ 168, 33 ] ], "l:635": [ [ 267, 33 ] ], "l:638": [ [ 72, 1 ], [ 888, 1 ], [ 891, 3 ], [ 896, 14 ], [ 911, 9 ], [ 1805, 3 ], [ 1809, 1 ], [ 1811, 1 ] ], "l:639": [ [ 1010, 1 ] ], "l:646": [ [ 1042, 1 ] ], "l:647": [ [ 1043, 1 ] ], "l:65": [ [ 164, 1 ] ], "l:658": [ [ 1073, 1 ] ], "l:659": [ [ 46, 3 ], [ 83, 1 ], [ 1644, 5 ], [ 1694, 1 ], [ 1697, 1 ], [ 1708, 22 ] ], "l:66": [ [ 234, 33 ] ], "l:664": [ [ 1069, 1 ] ], "l:670": [ [ 1008, 1 ] ], "l:679": [ [ 1071, 1 ] ], "l:680": [ [ 1636, 3 ], [ 1640, 1 ], [ 1643, 1 ], [ 1664, 17 ], [ 1688, 6 ], [ 1695, 2 ], [ 1702, 2 ], [ 1706, 1 ] ], "l:682": [ [ 137, 1 ] ], "l:683": [ [ 998, 1 ] ], "l:687": [ [ 1070, 1 ] ], "l:694": [ [ 51, 1 ] ], "l:695": [ [ 1078, 1 ] ], "l:696": [ [ 1003, 1 ] ], "l:7": [ [ 41, 1 ] ], "l:706": [ [ 1044, 1 ] ], "l:707": [ [ 1045, 1 ] ], "l:71": [ [ 1084, 1 ] ], "l:715": [ [ 1074, 1 ] ], "l:716": [ [ 120, 4 ], [ 1642, 1 ], [ 1649, 4 ], [ 1698, 1 ], [ 1707, 1 ], [ 1730, 22 ] ], "l:72": [ [ 165, 1 ] ], "l:726": [ [ 993, 1 ] ], "l:732": [ [ 846, 1 ] ], "l:733": [ [ 1011, 1 ] ], "l:74": [ [ 166, 1 ] ], "l:742": [ [ 52, 1 ] ], "l:744": [ [ 841, 1 ] ], "l:75": [ [ 300, 33 ] ], "l:755": [ [ 1836, 1 ] ], "l:756": [ [ 1047, 1 ] ], "l:763": [ [ 887, 1 ] ], "l:764": [ [ 1012, 1 ] ], "l:768": [ [ 878, 1 ], [ 885, 2 ], [ 1663, 1 ] ], "l:773": [ [ 881, 1 ] ], "l:774": [ [ 994, 1 ] ], "l:781": [ [ 997, 1 ] ], "l:786": [ [ 874, 3 ] ], "l:790": [ [ 53, 1 ] ], "l:792": [ [ 842, 1 ] ], "l:8": [ [ 1, 33 ] ], "l:80": [ [ 1006, 1 ] ], "l:803": [ [ 1048, 1 ] ], "l:804": [ [ 1049, 1 ] ], "l:81": [ [ 1007, 1 ] ], "l:812": [ [ 1075, 1 ] ], "l:813": [ [ 70, 1 ], [ 124, 13 ], [ 138, 1 ], [ 1653, 3 ], [ 1752, 15 ] ], "l:821": [ [ 882, 1 ] ], "l:822": [ [ 995, 1 ] ], "l:829": [ [ 996, 1 ] ], "l:834": [ [ 871, 3 ] ], "l:838": [ [ 54, 1 ] ], "l:840": [ [ 843, 1 ] ], "l:851": [ [ 1050, 1 ] ], "l:852": [ [ 1837, 1 ] ], "l:857": [ [ 201, 33 ] ], "l:860": [ [ 74, 1 ], [ 1118, 21 ], [ 1140, 11 ] ], "l:864": [ [ 37, 1 ] ], "l:866": [ [ 844, 1 ] ], "l:877": [ [ 1052, 1 ] ], "l:878": [ [ 1053, 1 ] ], "l:883": [ [ 399, 33 ] ], "l:886": [ [ 630, 33 ] ], "l:889": [ [ 333, 33 ] ], "l:894": [ [ 1054, 1 ] ], "l:895": [ [ 1055, 1 ] ], "l:900": [ [ 1151, 33 ] ], "l:903": [ [ 663, 33 ] ], "l:908": [ [ 167, 1 ] ], "l:912": [ [ 1076, 1 ] ], "l:913": [ [ 66, 1 ], [ 71, 1 ], [ 139, 13 ], [ 1656, 3 ], [ 1699, 1 ], [ 1767, 14 ] ], "l:918": [ [ 1072, 1 ] ], "l:924": [ [ 1056, 1 ] ], "l:925": [ [ 1057, 1 ] ], "l:930": [ [ 432, 33 ] ], "l:933": [ [ 696, 33 ] ], "l:939": [ [ 1077, 1 ] ], "l:940": [ [ 152, 9 ], [ 828, 8 ], [ 1659, 3 ], [ 1781, 9 ], [ 1791, 4 ] ], "l:948": [ [ 1058, 1 ] ], "l:949": [ [ 1059, 1 ] ], "l:954": [ [ 465, 33 ] ], "l:962": [ [ 1635, 1 ] ], "l:963": [ [ 1013, 1 ] ], "l:967": [ [ 865, 6 ] ], "l:972": [ [ 1060, 1 ] ], "l:973": [ [ 42, 1 ] ], "l:978": [ [ 498, 33 ] ], "l:981": [ [ 729, 33 ] ], "l:986": [ [ 1804, 1 ] ], "l:987": [ [ 73, 1 ] ], "l:99": [ [ 1080, 1 ] ], "l:991": [ [ 34, 1 ], [ 859, 1 ], [ 862, 3 ], [ 1799, 1 ] ], "l:996": [ [ 1062, 1 ] ], "l:997": [ [ 1066, 1 ] ] } }, { "key": "L4.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1002": [ [ 531, 33 ] ], "l:1005": [ [ 762, 33 ] ], "l:1010": [ [ 884, 1 ] ], "l:1015": [ [ 853, 2 ], [ 856, 3 ], [ 1798, 1 ] ], "l:1018": [ [ 1386, 1 ] ], "l:1021": [ [ 1065, 1 ] ], "l:1026": [ [ 564, 33 ] ], "l:1034": [ [ 883, 1 ] ], "l:1039": [ [ 847, 3 ], [ 851, 1 ], [ 1662, 1 ], [ 1797, 1 ] ], "l:1042": [ [ 1389, 1 ] ], "l:1044": [ [ 1840, 1 ] ], "l:1045": [ [ 1067, 1 ] ], "l:1048": [ [ 1976, 33 ] ], "l:1050": [ [ 597, 33 ] ], "l:1051": [ [ 2009, 33 ] ], "l:1053": [ [ 795, 33 ] ], "l:1054": [ [ 834, 2 ], [ 840, 1 ], [ 845, 1 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 889, 2 ], [ 894, 2 ], [ 910, 1 ], [ 920, 1 ], [ 930, 1 ], [ 935, 1 ], [ 940, 1 ], [ 945, 1 ], [ 950, 1 ], [ 955, 1 ], [ 960, 1 ], [ 965, 1 ], [ 970, 1 ], [ 980, 1 ], [ 1014, 1 ], [ 2042, 4 ] ], "l:121": [ [ 43, 1 ] ], "l:122": [ [ 1689, 1 ] ], "l:123": [ [ 48, 1 ] ], "l:132": [ [ 1680, 1 ] ], "l:133": [ [ 1139, 1 ] ], "l:134": [ [ 84, 1 ] ], "l:135": [ [ 1800, 4 ] ], "l:14": [ [ 1021, 1 ] ], "l:140": [ [ 85, 33 ] ], "l:155": [ [ 82, 1 ] ], "l:159": [ [ 1407, 1 ] ], "l:16": [ [ 1004, 1 ] ], "l:160": [ [ 1676, 1 ] ], "l:166": [ [ 1264, 1 ] ], "l:171": [ [ 1690, 1 ] ], "l:173": [ [ 986, 1 ] ], "l:179": [ [ 118, 1 ] ], "l:180": [ [ 64, 1 ], [ 1691, 3 ] ], "l:191": [ [ 55, 3 ] ], "l:195": [ [ 119, 1 ] ], "l:199": [ [ 68, 1 ], [ 1643, 1 ], [ 1703, 1 ], [ 1706, 1 ] ], "l:2": [ [ 1790, 1 ] ], "l:213": [ [ 1041, 1 ] ], "l:242": [ [ 59, 1 ] ], "l:260": [ [ 60, 1 ] ], "l:265": [ [ 1020, 1 ] ], "l:266": [ [ 1831, 1 ] ], "l:295": [ [ 1046, 1 ] ], "l:297": [ [ 1022, 1 ] ], "l:298": [ [ 1026, 1 ] ], "l:310": [ [ 45, 1 ] ], "l:32": [ [ 1215, 33 ] ], "l:329": [ [ 1051, 1 ] ], "l:331": [ [ 1024, 1 ] ], "l:335": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ] ], "l:337": [ [ 954, 1 ], [ 956, 4 ], [ 961, 4 ], [ 966, 4 ], [ 971, 9 ], [ 981, 1 ], [ 987, 1 ], [ 999, 4 ], [ 1818, 4 ], [ 1823, 1 ] ], "l:34": [ [ 1678, 1 ] ], "l:348": [ [ 988, 1 ] ], "l:357": [ [ 1833, 1 ] ], "l:369": [ [ 1061, 1 ] ], "l:379": [ [ 1824, 1 ] ], "l:389": [ [ 989, 1 ] ], "l:398": [ [ 1834, 1 ] ], "l:406": [ [ 1063, 1 ] ], "l:408": [ [ 1028, 1 ] ], "l:416": [ [ 1825, 1 ] ], "l:426": [ [ 990, 1 ] ], "l:434": [ [ 1822, 1 ] ], "l:441": [ [ 1826, 1 ] ], "l:446": [ [ 1558, 1 ] ], "l:462": [ [ 991, 1 ] ], "l:469": [ [ 982, 1 ] ], "l:477": [ [ 1827, 1 ] ], "l:482": [ [ 1432, 1 ] ], "l:485": [ [ 1033, 1 ] ], "l:488": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ] ], "l:500": [ [ 992, 1 ] ], "l:511": [ [ 1828, 1 ] ], "l:516": [ [ 1535, 1 ] ], "l:544": [ [ 1829, 1 ] ], "l:549": [ [ 1841, 1 ] ], "l:557": [ [ 47, 1 ], [ 1184, 31 ], [ 1675, 1 ] ], "l:569": [ [ 40, 1 ], [ 46, 1 ], [ 73, 1 ], [ 1003, 1 ], [ 1009, 5 ], [ 1015, 3 ], [ 1068, 1 ], [ 1079, 2 ], [ 1082, 3 ], [ 1636, 2 ], [ 1644, 1 ], [ 1668, 7 ], [ 1695, 2 ], [ 1702, 1 ], [ 1708, 1 ], [ 1843, 1 ] ], "l:570": [ [ 35, 1 ] ], "l:58": [ [ 75, 1 ], [ 1248, 16 ], [ 1265, 16 ] ], "l:583": [ [ 1830, 1 ] ], "l:588": [ [ 1457, 1 ] ], "l:590": [ [ 1038, 1 ] ], "l:591": [ [ 1039, 1 ] ], "l:597": [ [ 50, 1 ] ], "l:60": [ [ 1638, 1 ] ], "l:609": [ [ 36, 1 ] ], "l:624": [ [ 1586, 1 ] ], "l:626": [ [ 1040, 1 ] ], "l:627": [ [ 1835, 1 ] ], "l:630": [ [ 76, 1 ], [ 1380, 2 ], [ 1383, 3 ], [ 1387, 2 ], [ 1390, 17 ], [ 1408, 5 ], [ 1639, 1 ], [ 1683, 2 ] ], "l:633": [ [ 1413, 19 ], [ 1433, 13 ], [ 1685, 1 ] ], "l:635": [ [ 267, 33 ] ], "l:636": [ [ 58, 1 ], [ 1446, 11 ], [ 1458, 3 ], [ 1462, 17 ], [ 1686, 1 ] ], "l:638": [ [ 72, 1 ], [ 888, 1 ], [ 891, 3 ], [ 896, 14 ], [ 911, 9 ], [ 1805, 3 ], [ 1809, 1 ], [ 1811, 1 ] ], "l:644": [ [ 1844, 1 ] ], "l:658": [ [ 1073, 1 ] ], "l:662": [ [ 1645, 1 ], [ 1712, 3 ] ], "l:664": [ [ 1069, 1 ] ], "l:665": [ [ 67, 1 ] ], "l:67": [ [ 1281, 33 ] ], "l:670": [ [ 1008, 1 ] ], "l:677": [ [ 83, 1 ] ], "l:679": [ [ 1071, 1 ] ], "l:682": [ [ 137, 1 ] ], "l:683": [ [ 998, 1 ] ], "l:685": [ [ 1715, 3 ] ], "l:687": [ [ 1070, 1 ] ], "l:688": [ [ 1688, 1 ] ], "l:69": [ [ 1679, 1 ] ], "l:692": [ [ 38, 2 ], [ 164, 3 ], [ 828, 3 ], [ 832, 2 ], [ 836, 4 ], [ 841, 4 ], [ 983, 3 ], [ 994, 4 ], [ 1664, 4 ], [ 1792, 4 ] ], "l:693": [ [ 51, 1 ] ], "l:695": [ [ 1078, 1 ] ], "l:7": [ [ 41, 1 ] ], "l:704": [ [ 1332, 1 ] ], "l:706": [ [ 1044, 1 ] ], "l:707": [ [ 1045, 1 ] ], "l:715": [ [ 1074, 1 ] ], "l:724": [ [ 1724, 1 ] ], "l:725": [ [ 993, 1 ] ], "l:732": [ [ 846, 1 ] ], "l:738": [ [ 1647, 1 ], [ 1722, 2 ] ], "l:741": [ [ 71, 1 ], [ 150, 14 ], [ 831, 1 ], [ 1658, 4 ], [ 1778, 12 ], [ 1791, 1 ] ], "l:742": [ [ 52, 1 ] ], "l:753": [ [ 1461, 1 ] ], "l:755": [ [ 1836, 1 ] ], "l:756": [ [ 1047, 1 ] ], "l:763": [ [ 887, 1 ] ], "l:768": [ [ 878, 1 ], [ 885, 2 ], [ 1663, 1 ] ], "l:773": [ [ 881, 1 ] ], "l:78": [ [ 1023, 1 ] ], "l:786": [ [ 874, 3 ] ], "l:789": [ [ 66, 1 ], [ 135, 2 ], [ 138, 12 ], [ 1656, 2 ], [ 1699, 1 ], [ 1763, 15 ] ], "l:8": [ [ 1, 33 ] ], "l:801": [ [ 1081, 1 ] ], "l:803": [ [ 1048, 1 ] ], "l:812": [ [ 1075, 1 ] ], "l:821": [ [ 882, 1 ] ], "l:834": [ [ 871, 3 ] ], "l:837": [ [ 70, 1 ], [ 121, 14 ], [ 1642, 1 ], [ 1653, 3 ], [ 1698, 1 ], [ 1750, 13 ] ], "l:838": [ [ 54, 1 ] ], "l:851": [ [ 1050, 1 ] ], "l:852": [ [ 1837, 1 ] ], "l:857": [ [ 201, 33 ] ], "l:860": [ [ 74, 1 ], [ 1118, 21 ], [ 1140, 11 ] ], "l:863": [ [ 69, 1 ], [ 120, 1 ], [ 1648, 5 ], [ 1697, 1 ], [ 1707, 1 ], [ 1726, 24 ] ], "l:875": [ [ 1482, 1 ] ], "l:881": [ [ 1479, 3 ], [ 1483, 9 ], [ 1493, 19 ], [ 1687, 1 ], [ 1705, 1 ] ], "l:883": [ [ 399, 33 ] ], "l:884": [ [ 61, 1 ], [ 1512, 23 ], [ 1536, 9 ] ], "l:886": [ [ 630, 33 ] ], "l:887": [ [ 62, 1 ], [ 1545, 13 ], [ 1559, 19 ] ], "l:889": [ [ 333, 33 ] ], "l:892": [ [ 1357, 1 ] ], "l:894": [ [ 1054, 1 ] ], "l:895": [ [ 1055, 1 ] ], "l:898": [ [ 63, 1 ], [ 1578, 8 ], [ 1587, 24 ] ], "l:900": [ [ 1151, 33 ] ], "l:901": [ [ 65, 1 ], [ 1611, 24 ], [ 1641, 1 ], [ 1700, 1 ], [ 1808, 1 ], [ 1810, 1 ], [ 1812, 1 ], [ 1838, 2 ], [ 1842, 1 ] ], "l:903": [ [ 663, 33 ] ], "l:908": [ [ 167, 1 ] ], "l:912": [ [ 1076, 1 ] ], "l:916": [ [ 1646, 1 ], [ 1694, 1 ], [ 1718, 4 ] ], "l:919": [ [ 1640, 1 ] ], "l:922": [ [ 1492, 1 ] ], "l:930": [ [ 432, 33 ] ], "l:931": [ [ 1701, 1 ], [ 1845, 32 ] ], "l:933": [ [ 696, 33 ] ], "l:938": [ [ 1725, 1 ] ], "l:939": [ [ 1077, 1 ] ], "l:946": [ [ 1111, 1 ] ], "l:948": [ [ 1058, 1 ] ], "l:949": [ [ 1059, 1 ] ], "l:952": [ [ 1877, 33 ] ], "l:954": [ [ 465, 33 ] ], "l:96": [ [ 81, 1 ], [ 1709, 3 ] ], "l:962": [ [ 1635, 1 ] ], "l:967": [ [ 865, 6 ] ], "l:97": [ [ 1677, 1 ] ], "l:970": [ [ 1369, 1 ] ], "l:976": [ [ 1910, 33 ] ], "l:978": [ [ 498, 33 ] ], "l:979": [ [ 1943, 33 ] ], "l:981": [ [ 729, 33 ] ], "l:986": [ [ 1804, 1 ] ], "l:991": [ [ 34, 1 ], [ 859, 1 ], [ 862, 3 ], [ 1799, 1 ] ], "l:994": [ [ 1382, 1 ] ], "l:997": [ [ 1066, 1 ] ] } }, { "key": "L5.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1002": [ [ 531, 33 ] ], "l:1005": [ [ 762, 33 ] ], "l:1010": [ [ 884, 1 ] ], "l:1015": [ [ 853, 2 ], [ 856, 3 ], [ 1798, 1 ] ], "l:1018": [ [ 1386, 1 ] ], "l:1026": [ [ 564, 33 ] ], "l:1034": [ [ 883, 1 ] ], "l:1039": [ [ 847, 3 ], [ 851, 1 ], [ 1662, 1 ], [ 1797, 1 ] ], "l:1040": [ [ 896, 33 ] ], "l:1042": [ [ 1389, 1 ] ], "l:1046": [ [ 279, 33 ] ], "l:119": [ [ 323, 1 ] ], "l:12": [ [ 717, 33 ] ], "l:130": [ [ 320, 3 ] ], "l:132": [ [ 1680, 1 ] ], "l:133": [ [ 1139, 1 ] ], "l:134": [ [ 84, 1 ] ], "l:135": [ [ 1800, 4 ] ], "l:14": [ [ 1021, 1 ] ], "l:140": [ [ 85, 33 ] ], "l:155": [ [ 82, 1 ] ], "l:157": [ [ 319, 1 ] ], "l:158": [ [ 35, 1 ] ], "l:171": [ [ 1690, 1 ] ], "l:179": [ [ 118, 1 ] ], "l:180": [ [ 64, 1 ], [ 1691, 3 ] ], "l:191": [ [ 55, 3 ] ], "l:195": [ [ 119, 1 ] ], "l:199": [ [ 68, 1 ], [ 1643, 1 ], [ 1703, 1 ], [ 1706, 1 ] ], "l:2": [ [ 1790, 1 ] ], "l:213": [ [ 1041, 1 ] ], "l:242": [ [ 59, 1 ] ], "l:260": [ [ 60, 1 ] ], "l:265": [ [ 1020, 1 ] ], "l:266": [ [ 1831, 1 ] ], "l:295": [ [ 1046, 1 ] ], "l:30": [ [ 37, 1 ], [ 42, 1 ], [ 44, 1 ], [ 48, 2 ], [ 51, 4 ], [ 58, 1 ], [ 61, 2 ], [ 67, 1 ], [ 72, 1 ], [ 75, 6 ], [ 137, 1 ], [ 168, 12 ] ], "l:310": [ [ 45, 1 ] ], "l:329": [ [ 1051, 1 ] ], "l:335": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ] ], "l:34": [ [ 1678, 1 ] ], "l:348": [ [ 988, 1 ] ], "l:357": [ [ 1833, 1 ] ], "l:369": [ [ 1061, 1 ] ], "l:379": [ [ 1824, 1 ] ], "l:389": [ [ 989, 1 ] ], "l:398": [ [ 1834, 1 ] ], "l:406": [ [ 1063, 1 ] ], "l:416": [ [ 1825, 1 ] ], "l:426": [ [ 990, 1 ] ], "l:434": [ [ 1822, 1 ] ], "l:441": [ [ 1826, 1 ] ], "l:446": [ [ 1558, 1 ] ], "l:462": [ [ 991, 1 ] ], "l:469": [ [ 982, 1 ] ], "l:477": [ [ 1827, 1 ] ], "l:482": [ [ 1432, 1 ] ], "l:488": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ] ], "l:500": [ [ 992, 1 ] ], "l:511": [ [ 1828, 1 ] ], "l:516": [ [ 1535, 1 ] ], "l:544": [ [ 1829, 1 ] ], "l:549": [ [ 1841, 1 ] ], "l:555": [ [ 353, 33 ] ], "l:557": [ [ 47, 1 ], [ 1184, 31 ], [ 1675, 1 ] ], "l:567": [ [ 386, 33 ] ], "l:569": [ [ 40, 1 ], [ 46, 1 ], [ 73, 1 ], [ 1003, 1 ], [ 1009, 5 ], [ 1015, 3 ], [ 1068, 1 ], [ 1079, 2 ], [ 1082, 3 ], [ 1636, 2 ], [ 1644, 1 ], [ 1668, 7 ], [ 1695, 2 ], [ 1702, 1 ], [ 1708, 1 ], [ 1843, 1 ] ], "l:583": [ [ 1830, 1 ] ], "l:588": [ [ 1457, 1 ] ], "l:597": [ [ 50, 1 ] ], "l:60": [ [ 1638, 1 ] ], "l:609": [ [ 36, 1 ] ], "l:622": [ [ 750, 12 ], [ 795, 21 ] ], "l:624": [ [ 1586, 1 ] ], "l:628": [ [ 180, 21 ], [ 234, 12 ] ], "l:644": [ [ 1844, 1 ] ], "l:654": [ [ 1113, 1 ] ], "l:655": [ [ 935, 33 ] ], "l:657": [ [ 347, 4 ] ], "l:658": [ [ 1073, 1 ] ], "l:661": [ [ 312, 4 ] ], "l:670": [ [ 1008, 1 ] ], "l:675": [ [ 344, 3 ] ], "l:676": [ [ 83, 1 ] ], "l:683": [ [ 998, 1 ] ], "l:684": [ [ 316, 3 ] ], "l:689": [ [ 1028, 13 ], [ 1042, 4 ], [ 1047, 3 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 3 ] ], "l:69": [ [ 1679, 1 ] ], "l:690": [ [ 419, 13 ], [ 465, 20 ] ], "l:7": [ [ 41, 1 ] ], "l:704": [ [ 1332, 1 ] ], "l:714": [ [ 352, 1 ] ], "l:715": [ [ 1074, 1 ] ], "l:719": [ [ 340, 4 ] ], "l:724": [ [ 1724, 1 ] ], "l:736": [ [ 968, 14 ], [ 986, 2 ], [ 993, 1 ], [ 999, 4 ], [ 1004, 4 ], [ 1014, 1 ], [ 1018, 2 ], [ 1022, 1 ], [ 1024, 4 ] ], "l:738": [ [ 1647, 1 ], [ 1722, 2 ] ], "l:739": [ [ 485, 33 ] ], "l:741": [ [ 71, 1 ], [ 150, 14 ], [ 831, 1 ], [ 1658, 4 ], [ 1778, 12 ], [ 1791, 1 ] ], "l:753": [ [ 1461, 1 ] ], "l:763": [ [ 887, 1 ] ], "l:768": [ [ 878, 1 ], [ 885, 2 ], [ 1663, 1 ] ], "l:773": [ [ 881, 1 ] ], "l:78": [ [ 1023, 1 ] ], "l:786": [ [ 874, 3 ] ], "l:787": [ [ 518, 13 ], [ 597, 20 ] ], "l:8": [ [ 1, 33 ] ], "l:801": [ [ 1081, 1 ] ], "l:811": [ [ 351, 1 ] ], "l:812": [ [ 1075, 1 ] ], "l:816": [ [ 336, 4 ] ], "l:821": [ [ 882, 1 ] ], "l:834": [ [ 871, 3 ] ], "l:837": [ [ 70, 1 ], [ 121, 14 ], [ 1642, 1 ], [ 1653, 3 ], [ 1698, 1 ], [ 1750, 13 ] ], "l:849": [ [ 716, 1 ] ], "l:851": [ [ 1050, 1 ] ], "l:852": [ [ 1837, 1 ] ], "l:855": [ [ 617, 33 ] ], "l:857": [ [ 201, 33 ] ], "l:858": [ [ 650, 33 ] ], "l:860": [ [ 74, 1 ], [ 1118, 21 ], [ 1140, 11 ] ], "l:861": [ [ 683, 33 ] ], "l:863": [ [ 69, 1 ], [ 120, 1 ], [ 1648, 5 ], [ 1697, 1 ], [ 1707, 1 ], [ 1726, 24 ] ], "l:873": [ [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ] ], "l:879": [ [ 246, 33 ] ], "l:892": [ [ 1357, 1 ] ], "l:898": [ [ 63, 1 ], [ 1578, 8 ], [ 1587, 24 ] ], "l:901": [ [ 65, 1 ], [ 1611, 24 ], [ 1641, 1 ], [ 1700, 1 ], [ 1808, 1 ], [ 1810, 1 ], [ 1812, 1 ], [ 1838, 2 ], [ 1842, 1 ] ], "l:908": [ [ 167, 1 ] ], "l:909": [ [ 1067, 1 ], [ 1069, 4 ], [ 1078, 1 ], [ 1085, 26 ], [ 1112, 1 ] ], "l:911": [ [ 330, 6 ] ], "l:912": [ [ 1076, 1 ] ], "l:915": [ [ 929, 6 ] ], "l:916": [ [ 1646, 1 ], [ 1694, 1 ], [ 1718, 4 ] ], "l:922": [ [ 1492, 1 ] ], "l:930": [ [ 432, 33 ] ], "l:931": [ [ 1701, 1 ], [ 1845, 32 ] ], "l:938": [ [ 1725, 1 ] ], "l:939": [ [ 1077, 1 ] ], "l:943": [ [ 324, 6 ] ], "l:946": [ [ 1111, 1 ] ], "l:952": [ [ 1877, 33 ] ], "l:96": [ [ 81, 1 ], [ 1709, 3 ] ], "l:962": [ [ 1635, 1 ] ], "l:967": [ [ 865, 6 ] ], "l:97": [ [ 1677, 1 ] ], "l:970": [ [ 1369, 1 ] ], "l:976": [ [ 1910, 33 ] ], "l:979": [ [ 1943, 33 ] ], "l:986": [ [ 1804, 1 ] ], "l:991": [ [ 34, 1 ], [ 859, 1 ], [ 862, 3 ], [ 1799, 1 ] ], "l:994": [ [ 1382, 1 ] ] } }, { "key": "L6.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1002": [ [ 531, 33 ] ], "l:1005": [ [ 762, 33 ] ], "l:1008": [ [ 1791, 9 ], [ 1805, 3 ], [ 1809, 1 ], [ 1811, 1 ], [ 1813, 9 ], [ 1823, 1 ], [ 1832, 1 ], [ 1835, 3 ], [ 1840, 1 ], [ 1843, 1 ], [ 1976, 3 ] ], "l:1014": [ [ 233, 33 ] ], "l:1018": [ [ 1386, 1 ] ], "l:1026": [ [ 564, 33 ] ], "l:1032": [ [ 1979, 33 ] ], "l:1034": [ [ 883, 1 ] ], "l:1038": [ [ 266, 33 ] ], "l:1040": [ [ 896, 33 ] ], "l:117": [ [ 1187, 3 ] ], "l:12": [ [ 717, 33 ] ], "l:129": [ [ 1184, 3 ] ], "l:134": [ [ 84, 1 ] ], "l:135": [ [ 1800, 4 ] ], "l:138": [ [ 1190, 33 ] ], "l:140": [ [ 85, 33 ] ], "l:155": [ [ 82, 1 ] ], "l:156": [ [ 1183, 1 ] ], "l:171": [ [ 1690, 1 ] ], "l:177": [ [ 2045, 1 ] ], "l:179": [ [ 118, 1 ] ], "l:180": [ [ 64, 1 ], [ 1691, 3 ] ], "l:191": [ [ 55, 3 ] ], "l:195": [ [ 119, 1 ] ], "l:199": [ [ 68, 1 ], [ 1643, 1 ], [ 1703, 1 ], [ 1706, 1 ] ], "l:2": [ [ 1790, 1 ] ], "l:213": [ [ 1041, 1 ] ], "l:242": [ [ 59, 1 ] ], "l:260": [ [ 60, 1 ] ], "l:265": [ [ 1020, 1 ] ], "l:266": [ [ 1831, 1 ] ], "l:295": [ [ 1046, 1 ] ], "l:310": [ [ 45, 1 ] ], "l:329": [ [ 1051, 1 ] ], "l:335": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ] ], "l:34": [ [ 1678, 1 ] ], "l:348": [ [ 988, 1 ] ], "l:357": [ [ 1833, 1 ] ], "l:369": [ [ 1061, 1 ] ], "l:379": [ [ 1824, 1 ] ], "l:389": [ [ 989, 1 ] ], "l:398": [ [ 1834, 1 ] ], "l:406": [ [ 1063, 1 ] ], "l:416": [ [ 1825, 1 ] ], "l:426": [ [ 990, 1 ] ], "l:434": [ [ 1822, 1 ] ], "l:441": [ [ 1826, 1 ] ], "l:446": [ [ 1558, 1 ] ], "l:462": [ [ 991, 1 ] ], "l:469": [ [ 982, 1 ] ], "l:477": [ [ 1827, 1 ] ], "l:482": [ [ 1432, 1 ] ], "l:488": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ] ], "l:500": [ [ 992, 1 ] ], "l:511": [ [ 1828, 1 ] ], "l:516": [ [ 1535, 1 ] ], "l:544": [ [ 1829, 1 ] ], "l:549": [ [ 1841, 1 ] ], "l:555": [ [ 353, 33 ] ], "l:567": [ [ 386, 33 ] ], "l:583": [ [ 1830, 1 ] ], "l:588": [ [ 1457, 1 ] ], "l:597": [ [ 50, 1 ] ], "l:60": [ [ 1638, 1 ] ], "l:609": [ [ 36, 1 ] ], "l:622": [ [ 750, 12 ], [ 795, 21 ] ], "l:644": [ [ 1844, 1 ] ], "l:652": [ [ 1223, 33 ] ], "l:654": [ [ 1113, 1 ] ], "l:656": [ [ 299, 24 ], [ 324, 9 ] ], "l:668": [ [ 1423, 9 ], [ 1433, 24 ] ], "l:674": [ [ 698, 18 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 1 ] ], "l:69": [ [ 1679, 1 ] ], "l:690": [ [ 419, 13 ], [ 465, 20 ] ], "l:7": [ [ 41, 1 ] ], "l:704": [ [ 1332, 1 ] ], "l:712": [ [ 1256, 33 ] ], "l:718": [ [ 333, 18 ], [ 617, 15 ] ], "l:722": [ [ 1458, 3 ], [ 1462, 30 ] ], "l:724": [ [ 1724, 1 ] ], "l:736": [ [ 968, 14 ], [ 986, 2 ], [ 993, 1 ], [ 999, 4 ], [ 1004, 4 ], [ 1014, 1 ], [ 1018, 2 ], [ 1022, 1 ], [ 1024, 4 ] ], "l:737": [ [ 848, 2 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 15 ], [ 878, 1 ], [ 885, 2 ], [ 929, 6 ] ], "l:739": [ [ 485, 33 ] ], "l:753": [ [ 1461, 1 ] ], "l:761": [ [ 1289, 25 ], [ 1380, 2 ], [ 1383, 3 ], [ 1387, 3 ] ], "l:763": [ [ 887, 1 ] ], "l:767": [ [ 632, 33 ] ], "l:771": [ [ 1493, 33 ] ], "l:78": [ [ 1023, 1 ] ], "l:785": [ [ 983, 3 ], [ 994, 5 ], [ 1003, 1 ], [ 1009, 5 ], [ 1015, 3 ], [ 1021, 1 ], [ 1050, 1 ], [ 1068, 1 ], [ 1073, 5 ], [ 1079, 2 ], [ 1082, 3 ], [ 1114, 3 ] ], "l:787": [ [ 518, 13 ], [ 597, 20 ] ], "l:8": [ [ 1, 33 ] ], "l:801": [ [ 1081, 1 ] ], "l:809": [ [ 1390, 33 ] ], "l:815": [ [ 665, 33 ] ], "l:819": [ [ 1526, 9 ], [ 1536, 22 ], [ 1559, 2 ] ], "l:821": [ [ 882, 1 ] ], "l:833": [ [ 1117, 33 ] ], "l:837": [ [ 70, 1 ], [ 121, 14 ], [ 1642, 1 ], [ 1653, 3 ], [ 1698, 1 ], [ 1750, 13 ] ], "l:847": [ [ 2012, 33 ] ], "l:849": [ [ 716, 1 ] ], "l:853": [ [ 1150, 33 ] ], "l:873": [ [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ] ], "l:892": [ [ 1357, 1 ] ], "l:898": [ [ 63, 1 ], [ 1578, 8 ], [ 1587, 24 ] ], "l:901": [ [ 65, 1 ], [ 1611, 24 ], [ 1641, 1 ], [ 1700, 1 ], [ 1808, 1 ], [ 1810, 1 ], [ 1812, 1 ], [ 1838, 2 ], [ 1842, 1 ] ], "l:906": [ [ 1561, 17 ], [ 1586, 1 ], [ 1636, 2 ], [ 1639, 2 ], [ 1644, 9 ], [ 1656, 2 ] ], "l:910": [ [ 34, 2 ], [ 37, 4 ], [ 42, 3 ], [ 46, 4 ], [ 51, 4 ], [ 58, 1 ], [ 61, 2 ], [ 66, 2 ], [ 69, 1 ], [ 71, 10 ] ], "l:922": [ [ 1492, 1 ] ], "l:930": [ [ 432, 33 ] ], "l:931": [ [ 1701, 1 ], [ 1845, 32 ] ], "l:936": [ [ 1658, 19 ], [ 1680, 1 ], [ 1683, 7 ], [ 1694, 4 ], [ 1699, 1 ], [ 1702, 1 ] ], "l:942": [ [ 83, 1 ], [ 120, 1 ], [ 135, 31 ] ], "l:946": [ [ 1111, 1 ] ], "l:952": [ [ 1877, 33 ] ], "l:96": [ [ 81, 1 ], [ 1709, 3 ] ], "l:960": [ [ 1705, 1 ], [ 1707, 2 ], [ 1712, 12 ], [ 1726, 18 ] ], "l:962": [ [ 1635, 1 ] ], "l:966": [ [ 166, 1 ], [ 168, 32 ] ], "l:97": [ [ 1677, 1 ] ], "l:970": [ [ 1369, 1 ] ], "l:976": [ [ 1910, 33 ] ], "l:979": [ [ 1943, 33 ] ], "l:984": [ [ 1744, 6 ], [ 1763, 27 ] ], "l:986": [ [ 1804, 1 ] ], "l:990": [ [ 200, 33 ] ], "l:994": [ [ 1382, 1 ] ] } }, { "key": "L7.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1000": [ [ 1005, 15 ], [ 1021, 2 ], [ 1024, 16 ] ], "l:1002": [ [ 531, 33 ] ], "l:1003": [ [ 1040, 1 ], [ 1042, 4 ], [ 1047, 4 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 14 ] ], "l:1005": [ [ 762, 33 ] ], "l:1006": [ [ 1078, 3 ], [ 1082, 29 ], [ 1112, 1 ] ], "l:1018": [ [ 1386, 1 ] ], "l:1024": [ [ 1113, 33 ] ], "l:1026": [ [ 564, 33 ] ], "l:1030": [ [ 1146, 33 ] ], "l:1040": [ [ 896, 33 ] ], "l:115": [ [ 1800, 4 ] ], "l:117": [ [ 1187, 3 ] ], "l:12": [ [ 717, 33 ] ], "l:134": [ [ 84, 1 ] ], "l:138": [ [ 1190, 33 ] ], "l:152": [ [ 1179, 1 ] ], "l:153": [ [ 1180, 1 ] ], "l:155": [ [ 82, 1 ] ], "l:156": [ [ 1183, 1 ] ], "l:171": [ [ 1690, 1 ] ], "l:175": [ [ 35, 1 ], [ 37, 2 ] ], "l:177": [ [ 2045, 1 ] ], "l:178": [ [ 34, 1 ] ], "l:179": [ [ 118, 1 ] ], "l:180": [ [ 64, 1 ], [ 1691, 3 ] ], "l:191": [ [ 55, 3 ] ], "l:195": [ [ 119, 1 ] ], "l:199": [ [ 68, 1 ], [ 1643, 1 ], [ 1703, 1 ], [ 1706, 1 ] ], "l:2": [ [ 1790, 1 ] ], "l:202": [ [ 1184, 1 ] ], "l:203": [ [ 1185, 1 ] ], "l:205": [ [ 1186, 1 ] ], "l:213": [ [ 1041, 1 ] ], "l:228": [ [ 1181, 1 ] ], "l:230": [ [ 1182, 1 ] ], "l:232": [ [ 39, 2 ], [ 42, 2 ] ], "l:236": [ [ 1635, 1 ] ], "l:237": [ [ 1724, 1 ] ], "l:239": [ [ 1725, 1 ] ], "l:242": [ [ 59, 1 ] ], "l:260": [ [ 60, 1 ] ], "l:265": [ [ 1020, 1 ] ], "l:266": [ [ 1831, 1 ] ], "l:295": [ [ 1046, 1 ] ], "l:310": [ [ 45, 1 ] ], "l:329": [ [ 1051, 1 ] ], "l:335": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ] ], "l:34": [ [ 1678, 1 ] ], "l:348": [ [ 988, 1 ] ], "l:357": [ [ 1833, 1 ] ], "l:369": [ [ 1061, 1 ] ], "l:379": [ [ 1824, 1 ] ], "l:389": [ [ 989, 1 ] ], "l:398": [ [ 1834, 1 ] ], "l:406": [ [ 1063, 1 ] ], "l:416": [ [ 1825, 1 ] ], "l:426": [ [ 990, 1 ] ], "l:434": [ [ 1822, 1 ] ], "l:441": [ [ 1826, 1 ] ], "l:446": [ [ 1558, 1 ] ], "l:462": [ [ 991, 1 ] ], "l:469": [ [ 982, 1 ] ], "l:477": [ [ 1827, 1 ] ], "l:482": [ [ 1432, 1 ] ], "l:488": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ] ], "l:500": [ [ 992, 1 ] ], "l:511": [ [ 1828, 1 ] ], "l:516": [ [ 1535, 1 ] ], "l:544": [ [ 1829, 1 ] ], "l:549": [ [ 1841, 1 ] ], "l:555": [ [ 353, 33 ] ], "l:567": [ [ 386, 33 ] ], "l:583": [ [ 1830, 1 ] ], "l:588": [ [ 1457, 1 ] ], "l:596": [ [ 44, 1 ], [ 46, 4 ], [ 51, 4 ], [ 58, 1 ], [ 61, 2 ], [ 66, 2 ], [ 69, 1 ], [ 71, 10 ], [ 83, 1 ], [ 85, 7 ] ], "l:60": [ [ 1638, 1 ] ], "l:609": [ [ 36, 1 ] ], "l:622": [ [ 750, 12 ], [ 795, 21 ] ], "l:644": [ [ 1844, 1 ] ], "l:650": [ [ 92, 26 ], [ 120, 1 ], [ 135, 6 ] ], "l:666": [ [ 141, 33 ] ], "l:69": [ [ 1679, 1 ] ], "l:690": [ [ 419, 13 ], [ 465, 20 ] ], "l:7": [ [ 41, 1 ] ], "l:704": [ [ 1332, 1 ] ], "l:710": [ [ 174, 33 ] ], "l:720": [ [ 207, 33 ] ], "l:739": [ [ 485, 33 ] ], "l:753": [ [ 1461, 1 ] ], "l:759": [ [ 240, 33 ] ], "l:769": [ [ 273, 33 ] ], "l:78": [ [ 1023, 1 ] ], "l:787": [ [ 518, 13 ], [ 597, 20 ] ], "l:8": [ [ 1, 33 ] ], "l:801": [ [ 1081, 1 ] ], "l:807": [ [ 306, 33 ] ], "l:817": [ [ 339, 14 ], [ 617, 19 ] ], "l:835": [ [ 636, 33 ] ], "l:837": [ [ 70, 1 ], [ 121, 14 ], [ 1642, 1 ], [ 1653, 3 ], [ 1698, 1 ], [ 1750, 13 ] ], "l:847": [ [ 2012, 33 ] ], "l:873": [ [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ] ], "l:892": [ [ 1357, 1 ] ], "l:898": [ [ 63, 1 ], [ 1578, 8 ], [ 1587, 24 ] ], "l:901": [ [ 65, 1 ], [ 1611, 24 ], [ 1641, 1 ], [ 1700, 1 ], [ 1808, 1 ], [ 1810, 1 ], [ 1812, 1 ], [ 1838, 2 ], [ 1842, 1 ] ], "l:904": [ [ 669, 33 ] ], "l:906": [ [ 1561, 17 ], [ 1586, 1 ], [ 1636, 2 ], [ 1639, 2 ], [ 1644, 9 ], [ 1656, 2 ] ], "l:92": [ [ 1804, 1 ] ], "l:922": [ [ 1492, 1 ] ], "l:928": [ [ 702, 15 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ] ], "l:930": [ [ 432, 33 ] ], "l:931": [ [ 1701, 1 ], [ 1845, 32 ] ], "l:934": [ [ 853, 2 ], [ 856, 4 ], [ 862, 15 ], [ 878, 1 ], [ 881, 7 ], [ 929, 4 ] ], "l:946": [ [ 1111, 1 ] ], "l:952": [ [ 1877, 33 ] ], "l:958": [ [ 933, 33 ] ], "l:96": [ [ 81, 1 ], [ 1709, 3 ] ], "l:97": [ [ 1677, 1 ] ], "l:970": [ [ 1369, 1 ] ], "l:976": [ [ 1910, 33 ] ], "l:979": [ [ 1943, 33 ] ], "l:982": [ [ 966, 16 ], [ 983, 5 ], [ 993, 12 ] ], "l:994": [ [ 1382, 1 ] ] } }, { "key": "L8.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1018": [ [ 1386, 1 ] ], "l:1024": [ [ 1113, 33 ] ], "l:1029": [ [ 1068, 13 ], [ 1082, 20 ] ], "l:1030": [ [ 1146, 33 ] ], "l:1040": [ [ 896, 33 ] ], "l:12": [ [ 717, 33 ] ], "l:144": [ [ 1025, 1 ] ], "l:145": [ [ 1026, 1 ] ], "l:147": [ [ 1027, 1 ] ], "l:152": [ [ 1179, 1 ] ], "l:153": [ [ 1180, 1 ] ], "l:155": [ [ 82, 1 ] ], "l:156": [ [ 1183, 1 ] ], "l:169": [ [ 480, 3 ] ], "l:171": [ [ 1690, 1 ] ], "l:174": [ [ 477, 3 ] ], "l:179": [ [ 118, 1 ] ], "l:180": [ [ 64, 1 ], [ 1691, 3 ] ], "l:185": [ [ 979, 3 ], [ 983, 5 ], [ 993, 25 ] ], "l:191": [ [ 55, 3 ] ], "l:194": [ [ 1018, 1 ] ], "l:195": [ [ 119, 1 ] ], "l:197": [ [ 1019, 1 ] ], "l:198": [ [ 34, 2 ], [ 37, 4 ], [ 42, 2 ], [ 50, 1 ], [ 68, 1 ], [ 70, 1 ], [ 81, 1 ], [ 84, 1 ], [ 92, 20 ] ], "l:2": [ [ 1790, 1 ] ], "l:202": [ [ 1184, 1 ] ], "l:203": [ [ 1185, 1 ] ], "l:205": [ [ 1186, 1 ] ], "l:213": [ [ 1041, 1 ] ], "l:227": [ [ 1021, 1 ] ], "l:228": [ [ 1181, 1 ] ], "l:230": [ [ 1182, 1 ] ], "l:231": [ [ 112, 6 ], [ 120, 27 ] ], "l:236": [ [ 1635, 1 ] ], "l:237": [ [ 1724, 1 ] ], "l:239": [ [ 1725, 1 ] ], "l:242": [ [ 59, 1 ] ], "l:260": [ [ 60, 1 ] ], "l:265": [ [ 1020, 1 ] ], "l:266": [ [ 1831, 1 ] ], "l:295": [ [ 1046, 1 ] ], "l:303": [ [ 1102, 9 ], [ 1112, 1 ], [ 1187, 3 ], [ 1223, 20 ] ], "l:310": [ [ 45, 1 ] ], "l:329": [ [ 1051, 1 ] ], "l:335": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ] ], "l:34": [ [ 1678, 1 ] ], "l:348": [ [ 988, 1 ] ], "l:357": [ [ 1833, 1 ] ], "l:369": [ [ 1061, 1 ] ], "l:379": [ [ 1824, 1 ] ], "l:389": [ [ 989, 1 ] ], "l:398": [ [ 1834, 1 ] ], "l:406": [ [ 1063, 1 ] ], "l:416": [ [ 1825, 1 ] ], "l:426": [ [ 990, 1 ] ], "l:434": [ [ 1822, 1 ] ], "l:441": [ [ 1826, 1 ] ], "l:446": [ [ 1558, 1 ] ], "l:462": [ [ 991, 1 ] ], "l:469": [ [ 982, 1 ] ], "l:477": [ [ 1827, 1 ] ], "l:482": [ [ 1432, 1 ] ], "l:488": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ] ], "l:500": [ [ 992, 1 ] ], "l:511": [ [ 1828, 1 ] ], "l:516": [ [ 1535, 1 ] ], "l:526": [ [ 868, 1 ] ], "l:533": [ [ 1028, 1 ] ], "l:539": [ [ 1029, 1 ] ], "l:544": [ [ 1829, 1 ] ], "l:549": [ [ 1841, 1 ] ], "l:555": [ [ 353, 33 ] ], "l:567": [ [ 386, 33 ] ], "l:583": [ [ 1830, 1 ] ], "l:588": [ [ 1457, 1 ] ], "l:594": [ [ 516, 33 ] ], "l:596": [ [ 44, 1 ], [ 46, 4 ], [ 51, 4 ], [ 58, 1 ], [ 61, 2 ], [ 66, 2 ], [ 69, 1 ], [ 71, 10 ], [ 83, 1 ], [ 85, 7 ] ], "l:60": [ [ 1638, 1 ] ], "l:608": [ [ 483, 33 ] ], "l:609": [ [ 36, 1 ] ], "l:622": [ [ 750, 12 ], [ 795, 21 ] ], "l:642": [ [ 549, 33 ] ], "l:648": [ [ 180, 33 ] ], "l:69": [ [ 1679, 1 ] ], "l:7": [ [ 41, 1 ] ], "l:702": [ [ 582, 33 ] ], "l:708": [ [ 213, 33 ] ], "l:751": [ [ 615, 33 ] ], "l:757": [ [ 246, 33 ] ], "l:78": [ [ 1023, 1 ] ], "l:799": [ [ 648, 21 ], [ 702, 12 ] ], "l:8": [ [ 1, 33 ] ], "l:801": [ [ 1081, 1 ] ], "l:805": [ [ 279, 33 ] ], "l:847": [ [ 2012, 33 ] ], "l:873": [ [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ] ], "l:892": [ [ 1357, 1 ] ], "l:896": [ [ 869, 8 ], [ 878, 1 ], [ 881, 7 ], [ 929, 4 ], [ 966, 13 ] ], "l:898": [ [ 63, 1 ], [ 1578, 8 ], [ 1587, 24 ] ], "l:901": [ [ 65, 1 ], [ 1611, 24 ], [ 1641, 1 ], [ 1700, 1 ], [ 1808, 1 ], [ 1810, 1 ], [ 1812, 1 ], [ 1838, 2 ], [ 1842, 1 ] ], "l:904": [ [ 669, 33 ] ], "l:91": [ [ 1022, 1 ] ], "l:92": [ [ 1804, 1 ] ], "l:920": [ [ 714, 3 ], [ 762, 30 ] ], "l:926": [ [ 312, 33 ] ], "l:94": [ [ 1024, 1 ] ], "l:946": [ [ 1111, 1 ] ], "l:95": [ [ 147, 33 ] ], "l:952": [ [ 1877, 33 ] ], "l:957": [ [ 1030, 11 ], [ 1042, 4 ], [ 1047, 4 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 4 ] ], "l:958": [ [ 933, 33 ] ], "l:968": [ [ 792, 3 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 6 ] ], "l:974": [ [ 345, 8 ], [ 419, 25 ] ], "l:994": [ [ 1382, 1 ] ], "l:998": [ [ 444, 33 ] ] } }, { "key": "L9.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1018": [ [ 1386, 1 ] ], "l:1024": [ [ 1113, 33 ] ], "l:1027": [ [ 140, 3 ], [ 152, 6 ], [ 165, 3 ], [ 169, 2 ], [ 173, 1 ], [ 175, 2 ], [ 185, 1 ], [ 192, 3 ], [ 196, 1 ], [ 198, 4 ], [ 203, 1 ], [ 331, 1 ], [ 699, 3 ], [ 1018, 2 ] ], "l:1029": [ [ 1068, 13 ], [ 1082, 20 ] ], "l:1030": [ [ 1146, 33 ] ], "l:1040": [ [ 896, 33 ] ], "l:12": [ [ 717, 33 ] ], "l:144": [ [ 1025, 1 ] ], "l:145": [ [ 1026, 1 ] ], "l:147": [ [ 1027, 1 ] ], "l:149": [ [ 339, 4 ] ], "l:150": [ [ 34, 1 ] ], "l:167": [ [ 64, 1 ], [ 1691, 3 ] ], "l:169": [ [ 480, 3 ] ], "l:171": [ [ 1690, 1 ] ], "l:179": [ [ 118, 1 ] ], "l:183": [ [ 296, 33 ] ], "l:185": [ [ 979, 3 ], [ 983, 5 ], [ 993, 25 ] ], "l:191": [ [ 55, 3 ] ], "l:192": [ [ 35, 6 ], [ 42, 3 ], [ 46, 9 ], [ 58, 1 ], [ 61, 3 ], [ 65, 11 ] ], "l:2": [ [ 1790, 1 ] ], "l:202": [ [ 1184, 1 ] ], "l:203": [ [ 1185, 1 ] ], "l:205": [ [ 1186, 1 ] ], "l:207": [ [ 335, 4 ] ], "l:210": [ [ 1244, 33 ] ], "l:213": [ [ 1041, 1 ] ], "l:224": [ [ 1243, 1 ] ], "l:225": [ [ 76, 16 ], [ 109, 6 ], [ 119, 1 ], [ 133, 7 ], [ 143, 3 ] ], "l:236": [ [ 1635, 1 ] ], "l:237": [ [ 1724, 1 ] ], "l:239": [ [ 1725, 1 ] ], "l:241": [ [ 329, 1 ], [ 332, 3 ] ], "l:242": [ [ 59, 1 ] ], "l:260": [ [ 60, 1 ] ], "l:265": [ [ 1020, 1 ] ], "l:266": [ [ 1831, 1 ] ], "l:274": [ [ 212, 1 ] ], "l:295": [ [ 1046, 1 ] ], "l:301": [ [ 92, 2 ], [ 343, 1 ], [ 347, 6 ], [ 419, 20 ], [ 440, 4 ] ], "l:303": [ [ 1102, 9 ], [ 1112, 1 ], [ 1187, 3 ], [ 1223, 20 ] ], "l:310": [ [ 45, 1 ] ], "l:312": [ [ 245, 1 ] ], "l:329": [ [ 1051, 1 ] ], "l:335": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ] ], "l:34": [ [ 1678, 1 ] ], "l:348": [ [ 988, 1 ] ], "l:352": [ [ 246, 1 ] ], "l:357": [ [ 1833, 1 ] ], "l:369": [ [ 1061, 1 ] ], "l:379": [ [ 1824, 1 ] ], "l:389": [ [ 989, 1 ] ], "l:393": [ [ 244, 1 ] ], "l:398": [ [ 1834, 1 ] ], "l:406": [ [ 1063, 1 ] ], "l:416": [ [ 1825, 1 ] ], "l:426": [ [ 990, 1 ] ], "l:434": [ [ 1822, 1 ] ], "l:441": [ [ 1826, 1 ] ], "l:446": [ [ 1558, 1 ] ], "l:462": [ [ 991, 1 ] ], "l:469": [ [ 982, 1 ] ], "l:477": [ [ 1827, 1 ] ], "l:482": [ [ 1432, 1 ] ], "l:488": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ] ], "l:500": [ [ 992, 1 ] ], "l:511": [ [ 1828, 1 ] ], "l:516": [ [ 1535, 1 ] ], "l:526": [ [ 868, 1 ] ], "l:530": [ [ 345, 1 ] ], "l:531": [ [ 344, 1 ] ], "l:532": [ [ 172, 1 ], [ 223, 12 ], [ 237, 4 ], [ 247, 16 ] ], "l:537": [ [ 346, 1 ] ], "l:538": [ [ 263, 33 ] ], "l:544": [ [ 1829, 1 ] ], "l:549": [ [ 1841, 1 ] ], "l:555": [ [ 353, 33 ] ], "l:567": [ [ 386, 33 ] ], "l:583": [ [ 1830, 1 ] ], "l:588": [ [ 1457, 1 ] ], "l:594": [ [ 516, 33 ] ], "l:60": [ [ 1638, 1 ] ], "l:606": [ [ 94, 15 ], [ 115, 3 ], [ 120, 10 ], [ 241, 1 ], [ 330, 1 ], [ 477, 3 ] ], "l:608": [ [ 483, 33 ] ], "l:622": [ [ 750, 12 ], [ 795, 21 ] ], "l:642": [ [ 549, 33 ] ], "l:69": [ [ 1679, 1 ] ], "l:7": [ [ 41, 1 ] ], "l:702": [ [ 582, 33 ] ], "l:751": [ [ 615, 33 ] ], "l:78": [ [ 1023, 1 ] ], "l:799": [ [ 648, 21 ], [ 702, 12 ] ], "l:8": [ [ 1, 33 ] ], "l:847": [ [ 2012, 33 ] ], "l:873": [ [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ] ], "l:89": [ [ 146, 6 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 174, 1 ], [ 177, 8 ], [ 186, 6 ], [ 195, 1 ], [ 197, 1 ], [ 222, 1 ] ], "l:890": [ [ 202, 1 ], [ 204, 8 ], [ 213, 9 ], [ 235, 2 ], [ 1021, 2 ], [ 1024, 1 ], [ 1028, 2 ], [ 1081, 1 ], [ 1179, 5 ], [ 1190, 2 ] ], "l:892": [ [ 1357, 1 ] ], "l:896": [ [ 869, 8 ], [ 878, 1 ], [ 881, 7 ], [ 929, 4 ], [ 966, 13 ] ], "l:920": [ [ 714, 3 ], [ 762, 30 ] ], "l:946": [ [ 1111, 1 ] ], "l:952": [ [ 1877, 33 ] ], "l:955": [ [ 130, 3 ], [ 669, 30 ] ], "l:957": [ [ 1030, 11 ], [ 1042, 4 ], [ 1047, 4 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 4 ] ], "l:958": [ [ 933, 33 ] ], "l:968": [ [ 792, 3 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 6 ] ], "l:992": [ [ 242, 2 ], [ 1192, 31 ] ], "l:994": [ [ 1382, 1 ] ], "l:998": [ [ 444, 33 ] ] } }, { "key": "L10.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1018": [ [ 1386, 1 ] ], "l:1022": [ [ 118, 1 ], [ 513, 3 ], [ 869, 8 ], [ 878, 1 ], [ 881, 7 ], [ 930, 1 ], [ 966, 6 ], [ 975, 4 ], [ 1008, 1 ], [ 1012, 1 ] ], "l:1024": [ [ 1113, 33 ] ], "l:1027": [ [ 140, 3 ], [ 152, 6 ], [ 165, 3 ], [ 169, 2 ], [ 173, 1 ], [ 175, 2 ], [ 185, 1 ], [ 192, 3 ], [ 196, 1 ], [ 198, 4 ], [ 203, 1 ], [ 331, 1 ], [ 699, 3 ], [ 1018, 2 ] ], "l:1030": [ [ 1146, 33 ] ], "l:1040": [ [ 896, 33 ] ], "l:12": [ [ 717, 33 ] ], "l:144": [ [ 1025, 1 ] ], "l:145": [ [ 1026, 1 ] ], "l:147": [ [ 1027, 1 ] ], "l:148": [ [ 304, 23 ], [ 332, 3 ], [ 336, 7 ] ], "l:191": [ [ 55, 3 ] ], "l:192": [ [ 35, 6 ], [ 42, 3 ], [ 46, 9 ], [ 58, 1 ], [ 61, 3 ], [ 65, 11 ] ], "l:2": [ [ 1790, 1 ] ], "l:202": [ [ 1184, 1 ] ], "l:203": [ [ 1185, 1 ] ], "l:205": [ [ 1186, 1 ] ], "l:206": [ [ 34, 1 ], [ 64, 1 ], [ 172, 1 ], [ 223, 12 ], [ 254, 17 ], [ 328, 1 ] ], "l:208": [ [ 444, 33 ] ], "l:210": [ [ 1244, 33 ] ], "l:213": [ [ 1041, 1 ] ], "l:224": [ [ 1243, 1 ] ], "l:225": [ [ 76, 16 ], [ 109, 6 ], [ 119, 1 ], [ 133, 7 ], [ 143, 3 ] ], "l:236": [ [ 1635, 1 ] ], "l:237": [ [ 1724, 1 ] ], "l:239": [ [ 1725, 1 ] ], "l:240": [ [ 271, 33 ] ], "l:242": [ [ 59, 1 ] ], "l:260": [ [ 60, 1 ] ], "l:265": [ [ 1020, 1 ] ], "l:266": [ [ 1831, 1 ] ], "l:274": [ [ 212, 1 ] ], "l:295": [ [ 1046, 1 ] ], "l:301": [ [ 92, 2 ], [ 343, 1 ], [ 347, 6 ], [ 419, 20 ], [ 440, 4 ] ], "l:310": [ [ 45, 1 ] ], "l:312": [ [ 245, 1 ] ], "l:329": [ [ 1051, 1 ] ], "l:335": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ] ], "l:34": [ [ 1678, 1 ] ], "l:348": [ [ 988, 1 ] ], "l:352": [ [ 246, 1 ] ], "l:357": [ [ 1833, 1 ] ], "l:369": [ [ 1061, 1 ] ], "l:379": [ [ 1824, 1 ] ], "l:389": [ [ 989, 1 ] ], "l:393": [ [ 244, 1 ] ], "l:398": [ [ 1834, 1 ] ], "l:406": [ [ 1063, 1 ] ], "l:416": [ [ 1825, 1 ] ], "l:426": [ [ 990, 1 ] ], "l:434": [ [ 1822, 1 ] ], "l:441": [ [ 1826, 1 ] ], "l:446": [ [ 1558, 1 ] ], "l:462": [ [ 991, 1 ] ], "l:469": [ [ 982, 1 ] ], "l:477": [ [ 1827, 1 ] ], "l:482": [ [ 1432, 1 ] ], "l:488": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ] ], "l:500": [ [ 992, 1 ] ], "l:511": [ [ 1828, 1 ] ], "l:516": [ [ 1535, 1 ] ], "l:526": [ [ 868, 1 ] ], "l:528": [ [ 329, 1 ] ], "l:530": [ [ 345, 1 ] ], "l:531": [ [ 344, 1 ] ], "l:537": [ [ 346, 1 ] ], "l:544": [ [ 1829, 1 ] ], "l:549": [ [ 1841, 1 ] ], "l:555": [ [ 353, 33 ] ], "l:567": [ [ 386, 33 ] ], "l:583": [ [ 1830, 1 ] ], "l:588": [ [ 1457, 1 ] ], "l:594": [ [ 516, 33 ] ], "l:60": [ [ 1638, 1 ] ], "l:606": [ [ 94, 15 ], [ 115, 3 ], [ 120, 10 ], [ 241, 1 ], [ 330, 1 ], [ 477, 3 ] ], "l:622": [ [ 750, 12 ], [ 795, 21 ] ], "l:642": [ [ 549, 33 ] ], "l:69": [ [ 1679, 1 ] ], "l:7": [ [ 41, 1 ] ], "l:702": [ [ 582, 33 ] ], "l:751": [ [ 615, 33 ] ], "l:78": [ [ 1023, 1 ] ], "l:799": [ [ 648, 21 ], [ 702, 12 ] ], "l:847": [ [ 2012, 33 ] ], "l:873": [ [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ] ], "l:89": [ [ 146, 6 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 174, 1 ], [ 177, 8 ], [ 186, 6 ], [ 195, 1 ], [ 197, 1 ], [ 222, 1 ] ], "l:890": [ [ 202, 1 ], [ 204, 8 ], [ 213, 9 ], [ 235, 2 ], [ 1021, 2 ], [ 1024, 1 ], [ 1028, 2 ], [ 1081, 1 ], [ 1179, 5 ], [ 1190, 2 ] ], "l:920": [ [ 714, 3 ], [ 762, 30 ] ], "l:946": [ [ 1111, 1 ] ], "l:950": [ [ 480, 33 ] ], "l:952": [ [ 1877, 33 ] ], "l:955": [ [ 130, 3 ], [ 669, 30 ] ], "l:958": [ [ 933, 33 ] ], "l:968": [ [ 792, 3 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 6 ] ], "l:992": [ [ 242, 2 ], [ 1192, 31 ] ] } }, { "key": "L11.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1016": [ [ 170, 1 ], [ 175, 2 ], [ 929, 1 ], [ 931, 28 ], [ 999, 1 ] ], "l:1018": [ [ 1386, 1 ] ], "l:1022": [ [ 118, 1 ], [ 513, 3 ], [ 869, 8 ], [ 878, 1 ], [ 881, 7 ], [ 930, 1 ], [ 966, 6 ], [ 975, 4 ], [ 1008, 1 ], [ 1012, 1 ] ], "l:1040": [ [ 896, 33 ] ], "l:12": [ [ 717, 33 ] ], "l:141": [ [ 192, 1 ] ], "l:142": [ [ 64, 1 ], [ 300, 21 ], [ 322, 1 ], [ 324, 3 ], [ 328, 1 ], [ 332, 3 ], [ 336, 3 ] ], "l:189": [ [ 675, 3 ] ], "l:192": [ [ 35, 6 ], [ 42, 3 ], [ 46, 9 ], [ 58, 1 ], [ 61, 3 ], [ 65, 11 ] ], "l:2": [ [ 1790, 1 ] ], "l:200": [ [ 34, 1 ], [ 41, 1 ], [ 172, 2 ], [ 196, 1 ], [ 199, 1 ], [ 223, 12 ], [ 254, 13 ], [ 331, 1 ], [ 1150, 1 ] ], "l:208": [ [ 444, 33 ] ], "l:213": [ [ 1041, 1 ] ], "l:224": [ [ 1243, 1 ] ], "l:225": [ [ 76, 16 ], [ 109, 6 ], [ 119, 1 ], [ 133, 7 ], [ 143, 3 ] ], "l:233": [ [ 963, 1 ] ], "l:234": [ [ 267, 33 ] ], "l:242": [ [ 59, 1 ] ], "l:246": [ [ 678, 3 ] ], "l:254": [ [ 131, 2 ] ], "l:256": [ [ 671, 2 ] ], "l:258": [ [ 673, 2 ] ], "l:260": [ [ 60, 1 ] ], "l:265": [ [ 1020, 1 ] ], "l:266": [ [ 1831, 1 ] ], "l:274": [ [ 212, 1 ] ], "l:275": [ [ 962, 1 ] ], "l:279": [ [ 681, 3 ] ], "l:287": [ [ 344, 1 ], [ 1025, 1 ] ], "l:289": [ [ 130, 1 ], [ 200, 1 ] ], "l:291": [ [ 669, 2 ] ], "l:295": [ [ 1046, 1 ] ], "l:301": [ [ 92, 2 ], [ 343, 1 ], [ 347, 6 ], [ 419, 20 ], [ 440, 4 ] ], "l:310": [ [ 45, 1 ] ], "l:312": [ [ 245, 1 ] ], "l:313": [ [ 961, 1 ] ], "l:323": [ [ 341, 2 ] ], "l:325": [ [ 185, 1 ], [ 345, 1 ] ], "l:329": [ [ 1051, 1 ] ], "l:335": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ] ], "l:34": [ [ 1678, 1 ] ], "l:348": [ [ 988, 1 ] ], "l:352": [ [ 246, 1 ] ], "l:353": [ [ 960, 1 ] ], "l:357": [ [ 1833, 1 ] ], "l:365": [ [ 339, 2 ] ], "l:369": [ [ 1061, 1 ] ], "l:379": [ [ 1824, 1 ] ], "l:389": [ [ 989, 1 ] ], "l:393": [ [ 244, 1 ] ], "l:394": [ [ 959, 1 ] ], "l:398": [ [ 1834, 1 ] ], "l:406": [ [ 1063, 1 ] ], "l:416": [ [ 1825, 1 ] ], "l:426": [ [ 990, 1 ] ], "l:434": [ [ 1822, 1 ] ], "l:441": [ [ 1826, 1 ] ], "l:446": [ [ 1558, 1 ] ], "l:462": [ [ 991, 1 ] ], "l:469": [ [ 982, 1 ] ], "l:477": [ [ 1827, 1 ] ], "l:482": [ [ 1432, 1 ] ], "l:488": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ] ], "l:500": [ [ 992, 1 ] ], "l:511": [ [ 1828, 1 ] ], "l:516": [ [ 1535, 1 ] ], "l:526": [ [ 868, 1 ] ], "l:527": [ [ 964, 1 ] ], "l:528": [ [ 329, 1 ] ], "l:537": [ [ 346, 1 ] ], "l:544": [ [ 1829, 1 ] ], "l:549": [ [ 1841, 1 ] ], "l:555": [ [ 353, 33 ] ], "l:567": [ [ 386, 33 ] ], "l:583": [ [ 1830, 1 ] ], "l:588": [ [ 1457, 1 ] ], "l:594": [ [ 516, 33 ] ], "l:60": [ [ 1638, 1 ] ], "l:606": [ [ 94, 15 ], [ 115, 3 ], [ 120, 10 ], [ 241, 1 ], [ 330, 1 ], [ 477, 3 ] ], "l:622": [ [ 750, 12 ], [ 795, 21 ] ], "l:642": [ [ 549, 33 ] ], "l:69": [ [ 1679, 1 ] ], "l:702": [ [ 582, 33 ] ], "l:751": [ [ 615, 33 ] ], "l:78": [ [ 1023, 1 ] ], "l:799": [ [ 648, 21 ], [ 702, 12 ] ], "l:847": [ [ 2012, 33 ] ], "l:873": [ [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ] ], "l:88": [ [ 201, 1 ] ], "l:89": [ [ 146, 6 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 174, 1 ], [ 177, 8 ], [ 186, 6 ], [ 195, 1 ], [ 197, 1 ], [ 222, 1 ] ], "l:890": [ [ 202, 1 ], [ 204, 8 ], [ 213, 9 ], [ 235, 2 ], [ 1021, 2 ], [ 1024, 1 ], [ 1028, 2 ], [ 1081, 1 ], [ 1179, 5 ], [ 1190, 2 ] ], "l:920": [ [ 714, 3 ], [ 762, 30 ] ], "l:944": [ [ 140, 3 ], [ 152, 6 ], [ 165, 3 ], [ 169, 1 ], [ 193, 2 ], [ 684, 18 ] ], "l:946": [ [ 1111, 1 ] ], "l:950": [ [ 480, 33 ] ], "l:968": [ [ 792, 3 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 6 ] ], "l:992": [ [ 242, 2 ], [ 1192, 31 ] ] } }, { "key": "L12.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1016": [ [ 170, 1 ], [ 175, 2 ], [ 929, 1 ], [ 931, 28 ], [ 999, 1 ] ], "l:1040": [ [ 896, 33 ] ], "l:12": [ [ 717, 33 ] ], "l:141": [ [ 192, 1 ] ], "l:186": [ [ 35, 1 ], [ 228, 1 ], [ 327, 1 ], [ 869, 8 ], [ 878, 1 ], [ 883, 5 ], [ 965, 1 ], [ 972, 10 ], [ 984, 2 ], [ 1006, 2 ], [ 1009, 1 ] ], "l:188": [ [ 50, 2 ], [ 53, 5 ], [ 72, 12 ], [ 143, 9 ], [ 158, 3 ], [ 229, 1 ], [ 332, 1 ] ], "l:2": [ [ 1790, 1 ] ], "l:208": [ [ 444, 33 ] ], "l:213": [ [ 1041, 1 ] ], "l:223": [ [ 1243, 1 ] ], "l:233": [ [ 963, 1 ] ], "l:242": [ [ 59, 1 ] ], "l:243": [ [ 198, 1 ], [ 203, 1 ], [ 983, 1 ], [ 986, 2 ], [ 993, 6 ], [ 1000, 6 ], [ 1010, 9 ], [ 1032, 6 ], [ 1042, 1 ] ], "l:245": [ [ 34, 1 ], [ 36, 1 ], [ 39, 1 ], [ 41, 1 ], [ 43, 2 ], [ 49, 1 ], [ 161, 4 ], [ 168, 1 ], [ 171, 1 ], [ 177, 8 ], [ 195, 3 ], [ 199, 1 ], [ 222, 6 ], [ 231, 1 ], [ 255, 1 ] ], "l:252": [ [ 1084, 1 ] ], "l:254": [ [ 131, 2 ] ], "l:256": [ [ 671, 2 ] ], "l:258": [ [ 673, 2 ] ], "l:260": [ [ 60, 1 ] ], "l:265": [ [ 1020, 1 ] ], "l:266": [ [ 1831, 1 ] ], "l:273": [ [ 212, 1 ] ], "l:275": [ [ 962, 1 ] ], "l:276": [ [ 1026, 1 ], [ 1038, 3 ], [ 1043, 3 ], [ 1047, 2 ], [ 1052, 8 ], [ 1068, 13 ], [ 1082, 2 ], [ 1087, 1 ] ], "l:278": [ [ 67, 1 ], [ 173, 1 ], [ 230, 1 ], [ 254, 1 ], [ 256, 28 ], [ 1150, 1 ] ], "l:285": [ [ 1085, 1 ] ], "l:287": [ [ 344, 1 ], [ 1025, 1 ] ], "l:289": [ [ 130, 1 ], [ 200, 1 ] ], "l:291": [ [ 669, 2 ] ], "l:295": [ [ 1046, 1 ] ], "l:301": [ [ 92, 2 ], [ 343, 1 ], [ 347, 6 ], [ 419, 20 ], [ 440, 4 ] ], "l:308": [ [ 295, 1 ] ], "l:310": [ [ 45, 1 ] ], "l:312": [ [ 245, 1 ] ], "l:313": [ [ 961, 1 ] ], "l:315": [ [ 289, 1 ] ], "l:321": [ [ 1086, 1 ] ], "l:323": [ [ 341, 2 ] ], "l:325": [ [ 185, 1 ], [ 345, 1 ] ], "l:329": [ [ 1051, 1 ] ], "l:335": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ] ], "l:34": [ [ 1678, 1 ] ], "l:346": [ [ 290, 1 ] ], "l:353": [ [ 960, 1 ] ], "l:355": [ [ 174, 1 ] ], "l:357": [ [ 1833, 1 ] ], "l:363": [ [ 1030, 1 ] ], "l:365": [ [ 339, 2 ] ], "l:369": [ [ 1061, 1 ] ], "l:379": [ [ 1824, 1 ] ], "l:387": [ [ 291, 1 ] ], "l:396": [ [ 286, 1 ] ], "l:398": [ [ 1834, 1 ] ], "l:403": [ [ 1088, 1 ] ], "l:406": [ [ 1063, 1 ] ], "l:416": [ [ 1825, 1 ] ], "l:424": [ [ 58, 1 ] ], "l:426": [ [ 990, 1 ] ], "l:431": [ [ 287, 1 ] ], "l:434": [ [ 1822, 1 ] ], "l:441": [ [ 1826, 1 ] ], "l:446": [ [ 1558, 1 ] ], "l:454": [ [ 37, 1 ], [ 69, 1 ], [ 109, 6 ], [ 118, 2 ], [ 186, 2 ], [ 232, 1 ], [ 496, 18 ], [ 515, 1 ], [ 930, 1 ] ], "l:460": [ [ 293, 1 ] ], "l:467": [ [ 288, 1 ] ], "l:469": [ [ 982, 1 ] ], "l:477": [ [ 1827, 1 ] ], "l:482": [ [ 1432, 1 ] ], "l:488": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ] ], "l:498": [ [ 296, 1 ] ], "l:505": [ [ 294, 1 ] ], "l:511": [ [ 1828, 1 ] ], "l:516": [ [ 1535, 1 ] ], "l:524": [ [ 46, 1 ], [ 68, 1 ], [ 70, 2 ], [ 84, 8 ], [ 329, 1 ], [ 336, 3 ], [ 480, 16 ], [ 882, 1 ] ], "l:525": [ [ 868, 1 ] ], "l:527": [ [ 964, 1 ] ], "l:537": [ [ 346, 1 ] ], "l:544": [ [ 1829, 1 ] ], "l:549": [ [ 1841, 1 ] ], "l:555": [ [ 353, 33 ] ], "l:567": [ [ 386, 33 ] ], "l:583": [ [ 1830, 1 ] ], "l:588": [ [ 1457, 1 ] ], "l:594": [ [ 516, 33 ] ], "l:60": [ [ 1638, 1 ] ], "l:606": [ [ 94, 15 ], [ 115, 3 ], [ 120, 10 ], [ 241, 1 ], [ 330, 1 ], [ 477, 3 ] ], "l:620": [ [ 61, 5 ], [ 297, 10 ], [ 308, 4 ], [ 313, 4 ], [ 318, 3 ], [ 322, 1 ], [ 324, 3 ], [ 328, 1 ], [ 333, 2 ] ], "l:621": [ [ 52, 1 ] ], "l:622": [ [ 750, 12 ], [ 795, 21 ] ], "l:642": [ [ 549, 33 ] ], "l:69": [ [ 1679, 1 ] ], "l:702": [ [ 582, 33 ] ], "l:751": [ [ 615, 33 ] ], "l:78": [ [ 1023, 1 ] ], "l:799": [ [ 648, 21 ], [ 702, 12 ] ], "l:847": [ [ 2012, 33 ] ], "l:86": [ [ 38, 1 ], [ 40, 1 ], [ 66, 1 ], [ 133, 7 ], [ 188, 4 ], [ 233, 2 ], [ 331, 1 ], [ 675, 9 ], [ 966, 6 ], [ 1008, 1 ] ], "l:87": [ [ 201, 1 ] ], "l:873": [ [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ] ], "l:890": [ [ 202, 1 ], [ 204, 8 ], [ 213, 9 ], [ 235, 2 ], [ 1021, 2 ], [ 1024, 1 ], [ 1028, 2 ], [ 1081, 1 ], [ 1179, 5 ], [ 1190, 2 ] ], "l:920": [ [ 714, 3 ], [ 762, 30 ] ], "l:944": [ [ 140, 3 ], [ 152, 6 ], [ 165, 3 ], [ 169, 1 ], [ 193, 2 ], [ 684, 18 ] ], "l:968": [ [ 792, 3 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 6 ] ], "l:992": [ [ 242, 2 ], [ 1192, 31 ] ] } }, { "key": "L13.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1016": [ [ 170, 1 ], [ 175, 2 ], [ 929, 1 ], [ 931, 28 ], [ 999, 1 ] ], "l:1040": [ [ 896, 33 ] ], "l:12": [ [ 717, 33 ] ], "l:186": [ [ 35, 1 ], [ 228, 1 ], [ 327, 1 ], [ 869, 8 ], [ 878, 1 ], [ 883, 5 ], [ 965, 1 ], [ 972, 10 ], [ 984, 2 ], [ 1006, 2 ], [ 1009, 1 ] ], "l:2": [ [ 1790, 1 ] ], "l:208": [ [ 444, 33 ] ], "l:213": [ [ 1041, 1 ] ], "l:221": [ [ 1453, 4 ], [ 1458, 29 ] ], "l:222": [ [ 76, 1 ] ], "l:223": [ [ 1243, 1 ] ], "l:243": [ [ 198, 1 ], [ 203, 1 ], [ 983, 1 ], [ 986, 2 ], [ 993, 6 ], [ 1000, 6 ], [ 1010, 9 ], [ 1032, 6 ], [ 1042, 1 ] ], "l:250": [ [ 1419, 13 ], [ 1433, 20 ] ], "l:251": [ [ 78, 1 ] ], "l:252": [ [ 1084, 1 ] ], "l:265": [ [ 1020, 1 ] ], "l:266": [ [ 1831, 1 ] ], "l:271": [ [ 1386, 33 ] ], "l:272": [ [ 77, 1 ] ], "l:273": [ [ 212, 1 ] ], "l:276": [ [ 1026, 1 ], [ 1038, 3 ], [ 1043, 3 ], [ 1047, 2 ], [ 1052, 8 ], [ 1068, 13 ], [ 1082, 2 ], [ 1087, 1 ] ], "l:283": [ [ 1290, 24 ], [ 1332, 1 ], [ 1357, 1 ], [ 1369, 1 ], [ 1380, 6 ] ], "l:284": [ [ 79, 1 ] ], "l:285": [ [ 1085, 1 ] ], "l:295": [ [ 1046, 1 ] ], "l:301": [ [ 92, 2 ], [ 343, 1 ], [ 347, 6 ], [ 419, 20 ], [ 440, 4 ] ], "l:308": [ [ 295, 1 ] ], "l:314": [ [ 75, 1 ] ], "l:319": [ [ 1257, 33 ] ], "l:320": [ [ 80, 1 ] ], "l:321": [ [ 1086, 1 ] ], "l:329": [ [ 1051, 1 ] ], "l:335": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ] ], "l:34": [ [ 1678, 1 ] ], "l:345": [ [ 290, 1 ] ], "l:354": [ [ 52, 1 ] ], "l:357": [ [ 1833, 1 ] ], "l:361": [ [ 251, 3 ], [ 267, 1 ], [ 271, 2 ], [ 277, 1 ], [ 282, 1 ], [ 284, 2 ], [ 287, 1 ], [ 292, 1 ], [ 307, 1 ], [ 312, 1 ], [ 317, 1 ], [ 321, 1 ], [ 323, 1 ], [ 439, 1 ], [ 514, 1 ], [ 1027, 1 ], [ 1065, 3 ], [ 1092, 1 ], [ 1111, 1 ], [ 1146, 3 ], [ 1151, 1 ], [ 1238, 1 ], [ 1247, 2 ], [ 1253, 1 ] ], "l:362": [ [ 81, 1 ] ], "l:363": [ [ 1030, 1 ] ], "l:369": [ [ 1061, 1 ] ], "l:377": [ [ 247, 4 ], [ 1060, 1 ], [ 1062, 1 ], [ 1064, 1 ], [ 1226, 12 ], [ 1239, 4 ], [ 1244, 3 ], [ 1249, 4 ], [ 1254, 3 ] ], "l:378": [ [ 1824, 1 ] ], "l:386": [ [ 291, 1 ] ], "l:395": [ [ 72, 1 ] ], "l:398": [ [ 1834, 1 ] ], "l:401": [ [ 42, 1 ], [ 47, 2 ], [ 172, 1 ], [ 237, 4 ], [ 244, 1 ], [ 335, 1 ], [ 881, 1 ], [ 1019, 1 ], [ 1049, 2 ], [ 1149, 1 ], [ 1170, 9 ], [ 1184, 6 ], [ 1223, 3 ] ], "l:402": [ [ 82, 1 ] ], "l:403": [ [ 1088, 1 ] ], "l:406": [ [ 1063, 1 ] ], "l:414": [ [ 1131, 15 ], [ 1152, 18 ] ], "l:415": [ [ 1825, 1 ] ], "l:423": [ [ 58, 1 ] ], "l:430": [ [ 73, 1 ] ], "l:433": [ [ 1588, 1 ] ], "l:434": [ [ 1822, 1 ] ], "l:439": [ [ 1097, 14 ], [ 1112, 19 ] ], "l:440": [ [ 1826, 1 ] ], "l:446": [ [ 1558, 1 ] ], "l:452": [ [ 1520, 15 ], [ 1536, 18 ] ], "l:454": [ [ 37, 1 ], [ 69, 1 ], [ 109, 6 ], [ 118, 2 ], [ 186, 2 ], [ 232, 1 ], [ 496, 18 ], [ 515, 1 ], [ 930, 1 ] ], "l:459": [ [ 293, 1 ] ], "l:466": [ [ 74, 1 ] ], "l:469": [ [ 982, 1 ] ], "l:476": [ [ 1827, 1 ] ], "l:482": [ [ 1432, 1 ] ], "l:488": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ] ], "l:498": [ [ 296, 1 ] ], "l:504": [ [ 1589, 1 ] ], "l:505": [ [ 294, 1 ] ], "l:509": [ [ 39, 1 ], [ 41, 1 ], [ 43, 1 ], [ 177, 1 ], [ 192, 1 ], [ 195, 2 ], [ 199, 1 ], [ 201, 1 ], [ 670, 5 ], [ 959, 6 ], [ 988, 5 ], [ 1031, 1 ], [ 1089, 3 ], [ 1093, 4 ] ], "l:510": [ [ 1828, 1 ] ], "l:516": [ [ 1535, 1 ] ], "l:524": [ [ 46, 1 ], [ 68, 1 ], [ 70, 2 ], [ 84, 8 ], [ 329, 1 ], [ 336, 3 ], [ 480, 16 ], [ 882, 1 ] ], "l:536": [ [ 55, 3 ], [ 59, 2 ], [ 67, 1 ], [ 130, 3 ], [ 174, 1 ], [ 185, 1 ], [ 197, 1 ], [ 200, 1 ], [ 230, 1 ], [ 274, 3 ], [ 278, 4 ], [ 283, 1 ], [ 286, 1 ], [ 288, 2 ], [ 339, 4 ], [ 344, 2 ], [ 669, 1 ], [ 1025, 1 ] ], "l:537": [ [ 346, 1 ] ], "l:542": [ [ 49, 3 ], [ 53, 2 ], [ 173, 1 ], [ 223, 5 ], [ 231, 1 ], [ 245, 2 ], [ 254, 13 ], [ 268, 3 ], [ 273, 1 ], [ 868, 1 ], [ 1150, 1 ] ], "l:543": [ [ 1829, 1 ] ], "l:549": [ [ 1841, 1 ] ], "l:555": [ [ 353, 33 ] ], "l:567": [ [ 386, 33 ] ], "l:581": [ [ 34, 1 ], [ 36, 1 ], [ 44, 2 ], [ 83, 1 ], [ 143, 9 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 178, 7 ], [ 222, 1 ], [ 229, 1 ], [ 332, 1 ] ], "l:582": [ [ 1830, 1 ] ], "l:588": [ [ 1457, 1 ] ], "l:594": [ [ 516, 33 ] ], "l:60": [ [ 1638, 1 ] ], "l:606": [ [ 94, 15 ], [ 115, 3 ], [ 120, 10 ], [ 241, 1 ], [ 330, 1 ], [ 477, 3 ] ], "l:618": [ [ 1554, 4 ], [ 1559, 29 ] ], "l:620": [ [ 61, 5 ], [ 297, 10 ], [ 308, 4 ], [ 313, 4 ], [ 318, 3 ], [ 322, 1 ], [ 324, 3 ], [ 328, 1 ], [ 333, 2 ] ], "l:622": [ [ 750, 12 ], [ 795, 21 ] ], "l:642": [ [ 549, 33 ] ], "l:69": [ [ 1679, 1 ] ], "l:702": [ [ 582, 33 ] ], "l:751": [ [ 615, 33 ] ], "l:78": [ [ 1023, 1 ] ], "l:799": [ [ 648, 21 ], [ 702, 12 ] ], "l:84": [ [ 1487, 33 ] ], "l:847": [ [ 2012, 33 ] ], "l:86": [ [ 38, 1 ], [ 40, 1 ], [ 66, 1 ], [ 133, 7 ], [ 188, 4 ], [ 233, 2 ], [ 331, 1 ], [ 675, 9 ], [ 966, 6 ], [ 1008, 1 ] ], "l:873": [ [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ] ], "l:890": [ [ 202, 1 ], [ 204, 8 ], [ 213, 9 ], [ 235, 2 ], [ 1021, 2 ], [ 1024, 1 ], [ 1028, 2 ], [ 1081, 1 ], [ 1179, 5 ], [ 1190, 2 ] ], "l:920": [ [ 714, 3 ], [ 762, 30 ] ], "l:944": [ [ 140, 3 ], [ 152, 6 ], [ 165, 3 ], [ 169, 1 ], [ 193, 2 ], [ 684, 18 ] ], "l:968": [ [ 792, 3 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 6 ] ], "l:992": [ [ 242, 2 ], [ 1192, 31 ] ] } }, { "key": "L14.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1016": [ [ 170, 1 ], [ 175, 2 ], [ 929, 1 ], [ 931, 28 ], [ 999, 1 ] ], "l:1040": [ [ 896, 33 ] ], "l:12": [ [ 717, 33 ] ], "l:2": [ [ 1790, 1 ] ], "l:213": [ [ 1041, 1 ] ], "l:219": [ [ 1569, 12 ], [ 1590, 21 ] ], "l:243": [ [ 198, 1 ], [ 203, 1 ], [ 983, 1 ], [ 986, 2 ], [ 993, 6 ], [ 1000, 6 ], [ 1010, 9 ], [ 1032, 6 ], [ 1042, 1 ] ], "l:248": [ [ 1611, 27 ], [ 1639, 6 ] ], "l:265": [ [ 1020, 1 ] ], "l:266": [ [ 1831, 1 ] ], "l:269": [ [ 1645, 33 ] ], "l:276": [ [ 1026, 1 ], [ 1038, 3 ], [ 1043, 3 ], [ 1047, 2 ], [ 1052, 8 ], [ 1068, 13 ], [ 1082, 2 ], [ 1087, 1 ] ], "l:281": [ [ 1680, 1 ], [ 1683, 21 ], [ 1705, 11 ] ], "l:283": [ [ 1290, 24 ], [ 1332, 1 ], [ 1357, 1 ], [ 1369, 1 ], [ 1380, 6 ] ], "l:295": [ [ 1046, 1 ] ], "l:301": [ [ 92, 2 ], [ 343, 1 ], [ 347, 6 ], [ 419, 20 ], [ 440, 4 ] ], "l:306": [ [ 1501, 19 ], [ 1554, 4 ], [ 1559, 10 ] ], "l:307": [ [ 295, 1 ] ], "l:314": [ [ 75, 1 ] ], "l:319": [ [ 1257, 33 ] ], "l:329": [ [ 1051, 1 ] ], "l:335": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ] ], "l:34": [ [ 1678, 1 ] ], "l:343": [ [ 975, 7 ], [ 984, 2 ], [ 1006, 4 ], [ 1030, 1 ], [ 1084, 3 ], [ 1088, 1 ], [ 1243, 1 ], [ 1487, 14 ] ], "l:344": [ [ 290, 1 ] ], "l:357": [ [ 1833, 1 ] ], "l:359": [ [ 1716, 33 ] ], "l:369": [ [ 1061, 1 ] ], "l:377": [ [ 247, 4 ], [ 1060, 1 ], [ 1062, 1 ], [ 1064, 1 ], [ 1226, 12 ], [ 1239, 4 ], [ 1244, 3 ], [ 1249, 4 ], [ 1254, 3 ] ], "l:384": [ [ 676, 8 ], [ 869, 8 ], [ 878, 1 ], [ 883, 5 ], [ 930, 1 ], [ 965, 10 ] ], "l:385": [ [ 291, 1 ] ], "l:398": [ [ 1834, 1 ] ], "l:399": [ [ 1749, 33 ] ], "l:401": [ [ 42, 1 ], [ 47, 2 ], [ 172, 1 ], [ 237, 4 ], [ 244, 1 ], [ 335, 1 ], [ 881, 1 ], [ 1019, 1 ], [ 1049, 2 ], [ 1149, 1 ], [ 1170, 9 ], [ 1184, 6 ], [ 1223, 3 ] ], "l:406": [ [ 1063, 1 ] ], "l:412": [ [ 1782, 8 ], [ 1791, 25 ] ], "l:421": [ [ 517, 32 ], [ 675, 1 ] ], "l:422": [ [ 58, 1 ] ], "l:433": [ [ 1588, 1 ] ], "l:439": [ [ 1097, 14 ], [ 1112, 19 ] ], "l:446": [ [ 1558, 1 ] ], "l:452": [ [ 1520, 15 ], [ 1536, 18 ] ], "l:457": [ [ 467, 13 ], [ 496, 18 ], [ 515, 2 ] ], "l:458": [ [ 293, 1 ] ], "l:469": [ [ 982, 1 ] ], "l:475": [ [ 324, 5 ], [ 330, 2 ], [ 333, 2 ], [ 346, 1 ], [ 444, 23 ] ], "l:476": [ [ 1827, 1 ] ], "l:482": [ [ 1432, 1 ] ], "l:488": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ] ], "l:496": [ [ 188, 4 ], [ 212, 1 ], [ 228, 1 ], [ 232, 3 ], [ 241, 1 ], [ 294, 1 ], [ 297, 10 ], [ 308, 4 ], [ 313, 4 ], [ 318, 3 ], [ 322, 1 ] ], "l:497": [ [ 296, 1 ] ], "l:504": [ [ 1589, 1 ] ], "l:507": [ [ 1816, 11 ], [ 1828, 3 ], [ 1832, 1 ], [ 1835, 6 ], [ 1842, 12 ] ], "l:509": [ [ 39, 1 ], [ 41, 1 ], [ 43, 1 ], [ 177, 1 ], [ 192, 1 ], [ 195, 2 ], [ 199, 1 ], [ 201, 1 ], [ 670, 5 ], [ 959, 6 ], [ 988, 5 ], [ 1031, 1 ], [ 1089, 3 ], [ 1093, 4 ] ], "l:516": [ [ 1535, 1 ] ], "l:522": [ [ 1854, 33 ] ], "l:524": [ [ 46, 1 ], [ 68, 1 ], [ 70, 2 ], [ 84, 8 ], [ 329, 1 ], [ 336, 3 ], [ 480, 16 ], [ 882, 1 ] ], "l:534": [ [ 1887, 33 ] ], "l:536": [ [ 55, 3 ], [ 59, 2 ], [ 67, 1 ], [ 130, 3 ], [ 174, 1 ], [ 185, 1 ], [ 197, 1 ], [ 200, 1 ], [ 230, 1 ], [ 274, 3 ], [ 278, 4 ], [ 283, 1 ], [ 286, 1 ], [ 288, 2 ], [ 339, 4 ], [ 344, 2 ], [ 669, 1 ], [ 1025, 1 ] ], "l:540": [ [ 1920, 33 ] ], "l:549": [ [ 1841, 1 ] ], "l:555": [ [ 353, 33 ] ], "l:567": [ [ 386, 33 ] ], "l:579": [ [ 1953, 33 ] ], "l:581": [ [ 34, 1 ], [ 36, 1 ], [ 44, 2 ], [ 83, 1 ], [ 143, 9 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 178, 7 ], [ 222, 1 ], [ 229, 1 ], [ 332, 1 ] ], "l:588": [ [ 1457, 1 ] ], "l:592": [ [ 106, 24 ], [ 133, 7 ], [ 186, 2 ] ], "l:60": [ [ 1638, 1 ] ], "l:622": [ [ 750, 12 ], [ 795, 21 ] ], "l:642": [ [ 549, 33 ] ], "l:69": [ [ 1679, 1 ] ], "l:702": [ [ 582, 33 ] ], "l:751": [ [ 615, 33 ] ], "l:76": [ [ 1581, 6 ], [ 1986, 26 ], [ 2045, 1 ] ], "l:799": [ [ 648, 21 ], [ 702, 12 ] ], "l:82": [ [ 35, 1 ], [ 37, 2 ], [ 40, 1 ], [ 52, 1 ], [ 61, 6 ], [ 69, 1 ], [ 72, 3 ], [ 76, 2 ], [ 79, 4 ], [ 94, 12 ] ], "l:847": [ [ 2012, 33 ] ], "l:873": [ [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ] ], "l:890": [ [ 202, 1 ], [ 204, 8 ], [ 213, 9 ], [ 235, 2 ], [ 1021, 2 ], [ 1024, 1 ], [ 1028, 2 ], [ 1081, 1 ], [ 1179, 5 ], [ 1190, 2 ] ], "l:920": [ [ 714, 3 ], [ 762, 30 ] ], "l:944": [ [ 140, 3 ], [ 152, 6 ], [ 165, 3 ], [ 169, 1 ], [ 193, 2 ], [ 684, 18 ] ], "l:968": [ [ 792, 3 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 6 ] ], "l:992": [ [ 242, 2 ], [ 1192, 31 ] ] } }, { "key": "L15.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1016": [ [ 170, 1 ], [ 175, 2 ], [ 929, 1 ], [ 931, 28 ], [ 999, 1 ] ], "l:1040": [ [ 896, 33 ] ], "l:1057": [ [ 1854, 1 ], [ 1856, 1 ], [ 1859, 17 ], [ 1877, 1 ], [ 1889, 13 ] ], "l:12": [ [ 717, 33 ] ], "l:2": [ [ 1790, 1 ] ], "l:211": [ [ 1400, 32 ], [ 1913, 1 ] ], "l:213": [ [ 1041, 1 ] ], "l:217": [ [ 34, 14 ], [ 49, 18 ], [ 69, 1 ] ], "l:263": [ [ 1399, 1 ] ], "l:265": [ [ 1020, 1 ] ], "l:266": [ [ 1831, 1 ] ], "l:267": [ [ 1574, 7 ], [ 1587, 26 ] ], "l:269": [ [ 1645, 33 ] ], "l:276": [ [ 1026, 1 ], [ 1038, 3 ], [ 1043, 3 ], [ 1047, 2 ], [ 1052, 8 ], [ 1068, 13 ], [ 1082, 2 ], [ 1087, 1 ] ], "l:281": [ [ 1680, 1 ], [ 1683, 21 ], [ 1705, 11 ] ], "l:295": [ [ 1046, 1 ] ], "l:301": [ [ 92, 2 ], [ 343, 1 ], [ 347, 6 ], [ 419, 20 ], [ 440, 4 ] ], "l:304": [ [ 146, 6 ], [ 158, 7 ], [ 168, 1 ], [ 171, 4 ], [ 177, 9 ], [ 192, 1 ], [ 195, 5 ] ], "l:317": [ [ 48, 1 ], [ 200, 2 ], [ 203, 1 ], [ 222, 6 ], [ 229, 3 ], [ 237, 4 ], [ 244, 3 ], [ 254, 13 ] ], "l:319": [ [ 1257, 33 ] ], "l:329": [ [ 1051, 1 ] ], "l:335": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ] ], "l:34": [ [ 1678, 1 ] ], "l:341": [ [ 267, 27 ], [ 296, 1 ], [ 321, 1 ], [ 323, 1 ], [ 329, 1 ], [ 332, 1 ], [ 1858, 1 ] ], "l:359": [ [ 1716, 33 ] ], "l:369": [ [ 1061, 1 ] ], "l:375": [ [ 83, 1 ], [ 91, 1 ], [ 94, 2 ], [ 99, 2 ], [ 335, 8 ], [ 344, 2 ], [ 439, 1 ], [ 480, 16 ] ], "l:382": [ [ 101, 5 ], [ 514, 1 ], [ 669, 6 ], [ 868, 1 ], [ 881, 2 ], [ 959, 6 ], [ 982, 2 ], [ 986, 10 ] ], "l:399": [ [ 1749, 33 ] ], "l:406": [ [ 1063, 1 ] ], "l:412": [ [ 1782, 8 ], [ 1791, 25 ] ], "l:419": [ [ 996, 3 ], [ 1000, 6 ], [ 1010, 10 ], [ 1023, 1 ], [ 1025, 1 ], [ 1027, 1 ], [ 1031, 7 ], [ 1042, 1 ], [ 1049, 2 ], [ 1066, 1 ] ], "l:421": [ [ 517, 32 ], [ 675, 1 ] ], "l:437": [ [ 1065, 1 ], [ 1067, 1 ], [ 1089, 8 ], [ 1111, 1 ], [ 1131, 15 ], [ 1147, 2 ], [ 1150, 1 ], [ 1152, 3 ], [ 1855, 1 ] ], "l:439": [ [ 1097, 14 ], [ 1112, 19 ] ], "l:446": [ [ 1558, 1 ] ], "l:452": [ [ 1520, 15 ], [ 1536, 18 ] ], "l:455": [ [ 1155, 24 ], [ 1184, 6 ], [ 1223, 3 ] ], "l:457": [ [ 467, 13 ], [ 496, 18 ], [ 515, 2 ] ], "l:473": [ [ 251, 3 ], [ 1238, 1 ], [ 1247, 2 ], [ 1253, 1 ], [ 1290, 12 ], [ 1303, 1 ], [ 1305, 1 ], [ 1876, 1 ], [ 1878, 11 ] ], "l:475": [ [ 324, 5 ], [ 330, 2 ], [ 333, 2 ], [ 346, 1 ], [ 444, 23 ] ], "l:482": [ [ 1432, 1 ] ], "l:488": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ] ], "l:494": [ [ 1306, 8 ], [ 1332, 1 ], [ 1357, 1 ], [ 1369, 1 ], [ 1380, 19 ], [ 1903, 3 ] ], "l:507": [ [ 1816, 11 ], [ 1828, 3 ], [ 1832, 1 ], [ 1835, 6 ], [ 1842, 12 ] ], "l:514": [ [ 307, 1 ], [ 312, 1 ], [ 317, 1 ], [ 1146, 1 ], [ 1149, 1 ], [ 1151, 1 ], [ 1302, 1 ], [ 1304, 1 ], [ 1433, 24 ], [ 1458, 1 ] ], "l:520": [ [ 67, 2 ], [ 70, 13 ], [ 84, 7 ], [ 96, 3 ], [ 130, 3 ], [ 143, 3 ], [ 1643, 1 ], [ 1857, 1 ] ], "l:549": [ [ 1841, 1 ] ], "l:553": [ [ 1613, 25 ], [ 1639, 4 ], [ 1644, 1 ], [ 1827, 1 ], [ 1833, 2 ] ], "l:555": [ [ 353, 33 ] ], "l:567": [ [ 386, 33 ] ], "l:579": [ [ 1953, 33 ] ], "l:586": [ [ 1459, 28 ], [ 1569, 5 ] ], "l:592": [ [ 106, 24 ], [ 133, 7 ], [ 186, 2 ] ], "l:622": [ [ 750, 12 ], [ 795, 21 ] ], "l:642": [ [ 549, 33 ] ], "l:702": [ [ 582, 33 ] ], "l:751": [ [ 615, 33 ] ], "l:76": [ [ 1581, 6 ], [ 1986, 26 ], [ 2045, 1 ] ], "l:799": [ [ 648, 21 ], [ 702, 12 ] ], "l:847": [ [ 2012, 33 ] ], "l:873": [ [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ] ], "l:890": [ [ 202, 1 ], [ 204, 8 ], [ 213, 9 ], [ 235, 2 ], [ 1021, 2 ], [ 1024, 1 ], [ 1028, 2 ], [ 1081, 1 ], [ 1179, 5 ], [ 1190, 2 ] ], "l:920": [ [ 714, 3 ], [ 762, 30 ] ], "l:944": [ [ 140, 3 ], [ 152, 6 ], [ 165, 3 ], [ 169, 1 ], [ 193, 2 ], [ 684, 18 ] ], "l:968": [ [ 792, 3 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 6 ] ], "l:992": [ [ 242, 2 ], [ 1192, 31 ] ] } }, { "key": "L16.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:1016": [ [ 170, 1 ], [ 175, 2 ], [ 929, 1 ], [ 931, 28 ], [ 999, 1 ] ], "l:1040": [ [ 896, 33 ] ], "l:1057": [ [ 1854, 1 ], [ 1856, 1 ], [ 1859, 17 ], [ 1877, 1 ], [ 1889, 13 ] ], "l:12": [ [ 717, 33 ] ], "l:2": [ [ 1790, 1 ] ], "l:211": [ [ 1400, 32 ], [ 1913, 1 ] ], "l:261": [ [ 272, 33 ] ], "l:263": [ [ 1399, 1 ] ], "l:267": [ [ 1574, 7 ], [ 1587, 26 ] ], "l:293": [ [ 305, 2 ], [ 308, 4 ], [ 313, 4 ], [ 318, 23 ] ], "l:299": [ [ 34, 33 ] ], "l:327": [ [ 341, 33 ] ], "l:329": [ [ 1051, 1 ] ], "l:333": [ [ 67, 33 ] ], "l:367": [ [ 374, 33 ] ], "l:373": [ [ 100, 33 ] ], "l:404": [ [ 407, 33 ] ], "l:406": [ [ 1063, 1 ] ], "l:410": [ [ 133, 7 ], [ 143, 9 ], [ 158, 7 ], [ 168, 1 ], [ 171, 4 ], [ 177, 5 ] ], "l:444": [ [ 440, 33 ] ], "l:446": [ [ 1558, 1 ] ], "l:450": [ [ 182, 11 ], [ 195, 7 ], [ 203, 1 ], [ 212, 1 ], [ 222, 13 ] ], "l:480": [ [ 473, 33 ] ], "l:486": [ [ 237, 5 ], [ 244, 28 ] ], "l:514": [ [ 307, 1 ], [ 312, 1 ], [ 317, 1 ], [ 1146, 1 ], [ 1149, 1 ], [ 1151, 1 ], [ 1302, 1 ], [ 1304, 1 ], [ 1433, 24 ], [ 1458, 1 ] ], "l:547": [ [ 506, 33 ] ], "l:549": [ [ 1841, 1 ] ], "l:586": [ [ 1459, 28 ], [ 1569, 5 ] ], "l:622": [ [ 750, 12 ], [ 795, 21 ] ], "l:642": [ [ 549, 33 ] ], "l:702": [ [ 582, 33 ] ], "l:751": [ [ 615, 33 ] ], "l:76": [ [ 1581, 6 ], [ 1986, 26 ], [ 2045, 1 ] ], "l:799": [ [ 648, 21 ], [ 702, 12 ] ], "l:847": [ [ 2012, 33 ] ], "l:873": [ [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ] ], "l:890": [ [ 202, 1 ], [ 204, 8 ], [ 213, 9 ], [ 235, 2 ], [ 1021, 2 ], [ 1024, 1 ], [ 1028, 2 ], [ 1081, 1 ], [ 1179, 5 ], [ 1190, 2 ] ], "l:920": [ [ 714, 3 ], [ 762, 30 ] ], "l:944": [ [ 140, 3 ], [ 152, 6 ], [ 165, 3 ], [ 169, 1 ], [ 193, 2 ], [ 684, 18 ] ], "l:968": [ [ 792, 3 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 6 ] ], "l:992": [ [ 242, 2 ], [ 1192, 31 ] ] } }, { "key": "L17.post", "nodes": { "l:0": [ [ 1, 33 ] ], "l:10": [ [ 38, 1 ], [ 40, 2 ], [ 43, 1 ], [ 48, 1 ], [ 61, 3 ], [ 107, 18 ], [ 128, 1 ], [ 130, 3 ], [ 327, 1 ], [ 330, 1 ], [ 391, 1 ] ] } }, { "key": "output", "nodes": { "l:0": [ [ 1, 33 ] ] } } ], "width": 2048 }, "runtime_contract": { "causal_attention": true, "dtype": "float32", "generation": { "do_sample": false, "greedy": true }, "recommended_attention_implementation_for_analysis": "eager", "unsupported_inferences": [ "Static graph ancestry is not dynamic causal relevance.", "Unused-on-one-input activations require an input-specific trace.", "Other numerical precisions are outside the exactness contract." ] }, "schedule": { "assignment": [ { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": null, "node": "l:8", "routing": null }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 0, "node": "l:26", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:28", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:23", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:24", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:27", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:21", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 0, "node": "l:20", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:19", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:18", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:17", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:44", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:46", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:48", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:43", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:45", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:47", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "mlp", "layer": 2, "node": "l:41", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:7", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:52", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:53", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:55", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:50", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "mlp", "layer": 2, "node": "l:49", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:39", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:36", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:37", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:56", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 4, "node": "l:34", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:62", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:63", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:65", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 4, "node": "l:60", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:71", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:72", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:74", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 4, "node": "l:69", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:112", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:125", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:126", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:128", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:123", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 0, "node": "l:121", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:122", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 5, "node": "l:119", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 2, "node": "l:173", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 4, "node": "l:171", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:109", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:110", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:113", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:107", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:114", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 0, "node": "l:106", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:105", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:104", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:103", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:132", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:133", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:130", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:129", "routing": "attn" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 6, "node": "l:117", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 0, "node": "l:134", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 0, "node": "l:137", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 1, "node": "l:135", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:115", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 0, "node": "l:140", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "mlp", "layer": 6, "node": "l:138", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:162", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:163", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:165", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:160", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:166", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:159", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:158", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:157", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 6, "node": "l:156", "routing": "attn" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 6, "node": "l:177", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:178", "routing": "attn" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:175", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:174", "routing": "attn" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 8, "node": "l:169", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 0, "node": "l:179", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 0, "node": "l:182", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 4, "node": "l:180", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 9, "node": "l:167", "routing": "attn" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 8, "node": "l:185", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 9, "node": "l:183", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:99", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:100", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:102", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 4, "node": "l:97", "routing": "attn" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 7, "node": "l:152", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 7, "node": "l:153", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 0, "node": "l:155", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 9, "node": "l:150", "routing": "attn" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:198", "routing": "attn" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:194", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 0, "node": "l:195", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:197", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 9, "node": "l:192", "routing": "attn" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:231", "routing": "attn" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:227", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 7, "node": "l:228", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 7, "node": "l:230", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 9, "node": "l:225", "routing": "attn" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:95", "routing": "attn" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:91", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 7, "node": "l:92", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:94", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 9, "node": "l:89", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:206", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 7, "node": "l:202", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 7, "node": "l:203", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 7, "node": "l:205", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 11, "node": "l:200", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:240", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 7, "node": "l:236", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 7, "node": "l:237", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 7, "node": "l:239", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 11, "node": "l:234", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:148", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 8, "node": "l:144", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 8, "node": "l:145", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 8, "node": "l:147", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 11, "node": "l:142", "routing": "attn" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 9, "node": "l:274", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 9, "node": "l:224", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:88", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:275", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:233", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:141", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:87", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:223", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:273", "routing": "attn" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 0, "node": "l:242", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:222", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:272", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 0, "node": "l:340", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:339", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:338", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:381", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:380", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 1, "node": "l:379", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:378", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:418", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:417", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 1, "node": "l:416", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:415", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:493", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:492", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:491", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 8, "node": "l:526", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:533", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 9, "node": "l:532", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 9, "node": "l:531", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 9, "node": "l:530", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 10, "node": "l:528", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:527", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:525", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:559", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:566", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:565", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:564", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:563", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:561", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:560", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:558", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:598", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:605", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:604", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:603", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:602", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:600", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:599", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 3, "node": "l:597", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:664", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:665", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:662", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:661", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 5, "node": "l:655", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 0, "node": "l:660", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:659", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:658", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:656", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:652", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:717", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:716", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:715", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:718", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:712", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:766", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:765", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:764", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:767", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:761", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 0, "node": "l:814", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:813", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:812", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:815", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:809", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:57", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 0, "node": "l:632", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 0, "node": "l:857", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 9, "node": "l:393", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 9, "node": "l:352", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 9, "node": "l:312", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:394", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:353", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:313", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "mlp", "layer": 0, "node": "l:191", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 11, "node": "l:189", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:188", "routing": "attn" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 11, "node": "l:246", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:245", "routing": "attn" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 11, "node": "l:279", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:278", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 0, "node": "l:358", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:311", "routing": "attn" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 0, "node": "l:310", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 12, "node": "l:308", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:315", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:314", "routing": "attn" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:307", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:351", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:350", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:349", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 2, "node": "l:348", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:346", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:355", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:354", "routing": "attn" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:345", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 1, "node": "l:357", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 14, "node": "l:344", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:392", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:391", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:390", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 2, "node": "l:389", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:387", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:396", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:395", "routing": "attn" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:386", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 1, "node": "l:398", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 14, "node": "l:385", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:429", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:428", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:427", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 2, "node": "l:426", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:424", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:431", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:436", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:435", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 1, "node": "l:434", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:430", "routing": "attn" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:423", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:433", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 14, "node": "l:422", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:465", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:464", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:463", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 2, "node": "l:462", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:460", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:467", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:472", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:471", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:470", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:466", "routing": "attn" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:459", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 2, "node": "l:469", "routing": "attn" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 14, "node": "l:458", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:503", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:502", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:501", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 2, "node": "l:500", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 12, "node": "l:498", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:505", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:504", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 14, "node": "l:497", "routing": "attn" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:539", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 9, "node": "l:538", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 9, "node": "l:537", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:571", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:578", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:577", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:576", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 0, "node": "l:575", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:573", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:572", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:570", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:610", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:617", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:616", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:615", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 0, "node": "l:614", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:612", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:611", "routing": "attn" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 3, "node": "l:609", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:687", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:688", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:685", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:684", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 5, "node": "l:736", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 5, "node": "l:689", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 0, "node": "l:681", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:680", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:679", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:682", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:677", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:673", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:672", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:671", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 2, "node": "l:670", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:676", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:674", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:668", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:730", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:729", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:728", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 0, "node": "l:731", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 2, "node": "l:726", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 0, "node": "l:735", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:734", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:733", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:725", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:737", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 4, "node": "l:724", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:722", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:779", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:778", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:777", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:780", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:775", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:784", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:783", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:782", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:774", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:785", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 3, "node": "l:773", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:771", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:827", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:826", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:825", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 0, "node": "l:828", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:823", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:832", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:831", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:830", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:822", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:833", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 3, "node": "l:821", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:819", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 0, "node": "l:66", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:635", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:860", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:365", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:323", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:287", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:325", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:289", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:254", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:291", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:256", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 11, "node": "l:258", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:621", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:252", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:285", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:321", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:363", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:403", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 0, "node": "l:260", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:251", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:284", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:320", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:362", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:402", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:443", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:442", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 1, "node": "l:441", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:440", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:479", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:478", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 1, "node": "l:477", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:476", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:513", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:512", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 1, "node": "l:511", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:510", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:546", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:545", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 1, "node": "l:544", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:543", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:585", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:584", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 1, "node": "l:583", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:582", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:641", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:640", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:639", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:694", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:701", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:700", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:699", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:698", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:696", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:695", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:693", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:743", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:750", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:749", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:748", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:747", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:745", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:744", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:742", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:791", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:798", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:797", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:796", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:795", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:793", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:792", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:790", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:839", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:846", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:845", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:844", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:843", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:841", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:840", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:838", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:865", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:872", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:871", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:870", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 0, "node": "l:869", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:867", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:866", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:864", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:918", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:919", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:916", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:915", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 5, "node": "l:909", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 0, "node": "l:914", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:913", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:912", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:910", "routing": "attn" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 6, "node": "l:906", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 1, "node": "l:941", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:940", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:939", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:942", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:936", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:965", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:964", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:963", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:966", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:960", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 0, "node": "l:989", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:988", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:987", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:990", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:984", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 0, "node": "l:1013", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:1012", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:1011", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:1014", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:1008", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 0, "node": "l:1037", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:1036", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:1035", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:1038", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:1032", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 0, "node": "l:75", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:889", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 0, "node": "l:1056", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:883", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:900", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 0, "node": "l:930", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:954", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:978", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 0, "node": "l:1002", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 0, "node": "l:1026", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:1050", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:886", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:903", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:933", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 8, "node": "l:957", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:981", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 0, "node": "l:1005", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 8, "node": "l:1029", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:1053", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:32", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:58", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:67", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:30", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:84", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:186", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 10, "node": "l:208", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 14, "node": "l:82", "routing": "attn" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 14, "node": "l:219", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 12, "node": "l:243", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 14, "node": "l:248", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:217", "routing": "attn" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 14, "node": "l:269", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 12, "node": "l:276", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 14, "node": "l:281", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 15, "node": "l:267", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 9, "node": "l:301", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:304", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:317", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:299", "routing": "attn" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 4, "node": "l:335", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:341", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 14, "node": "l:359", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:333", "routing": "attn" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:375", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:382", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 14, "node": "l:399", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:373", "routing": "attn" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 14, "node": "l:412", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:419", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:437", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:410", "routing": "attn" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 13, "node": "l:452", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:455", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:473", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:450", "routing": "attn" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 4, "node": "l:488", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:494", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 14, "node": "l:507", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:486", "routing": "attn" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 14, "node": "l:522", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 14, "node": "l:534", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 14, "node": "l:540", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:520", "routing": "attn" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 5, "node": "l:555", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 5, "node": "l:567", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 14, "node": "l:579", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 15, "node": "l:553", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 8, "node": "l:594", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 9, "node": "l:606", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 13, "node": "l:618", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 14, "node": "l:592", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:630", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:633", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:636", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:628", "routing": "attn" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:650", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:666", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 5, "node": "l:690", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:648", "routing": "attn" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:710", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:720", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 5, "node": "l:739", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:708", "routing": "attn" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:759", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:769", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 5, "node": "l:787", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:757", "routing": "attn" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:807", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:817", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:835", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:805", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:855", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:858", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:861", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:853", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:881", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:884", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:887", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:879", "routing": "attn" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 4, "node": "l:898", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 4, "node": "l:901", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 7, "node": "l:904", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 8, "node": "l:896", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:928", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 4, "node": "l:931", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:934", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:926", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 4, "node": "l:952", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 9, "node": "l:955", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 7, "node": "l:958", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 10, "node": "l:950", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "mlp", "layer": 4, "node": "l:976", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 4, "node": "l:979", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:982", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:974", "routing": "attn" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:1000", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:1003", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:1006", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 8, "node": "l:998", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 7, "node": "l:1024", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 9, "node": "l:1027", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 7, "node": "l:1030", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 10, "node": "l:1022", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:1048", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:1051", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:1054", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:1046", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:4", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:5", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:9", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 1, "node": "l:2", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:16", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:29", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:14", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 5, "node": "l:12", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:80", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:81", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 4, "node": "l:78", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 14, "node": "l:76", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:215", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:216", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 4, "node": "l:213", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 15, "node": "l:211", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 3, "node": "l:265", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 3, "node": "l:266", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 15, "node": "l:263", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:261", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:297", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:298", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 4, "node": "l:295", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:293", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:331", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:332", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 4, "node": "l:329", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:327", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:371", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:372", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 4, "node": "l:369", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:367", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:408", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:409", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 4, "node": "l:406", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:404", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:448", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:449", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 4, "node": "l:446", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:444", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:484", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:485", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 4, "node": "l:482", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:480", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:518", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:519", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 4, "node": "l:516", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 15, "node": "l:514", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:551", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:552", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 4, "node": "l:549", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:547", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:590", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:591", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 4, "node": "l:588", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 15, "node": "l:586", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:626", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:627", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:624", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 5, "node": "l:622", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:646", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:647", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "mlp", "layer": 4, "node": "l:644", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 8, "node": "l:642", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:706", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:707", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "mlp", "layer": 4, "node": "l:704", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 8, "node": "l:702", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:755", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:756", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "mlp", "layer": 4, "node": "l:753", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 8, "node": "l:751", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:803", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:804", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 4, "node": "l:801", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 8, "node": "l:799", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:851", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:852", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 5, "node": "l:849", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 6, "node": "l:847", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:877", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:878", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:875", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 5, "node": "l:873", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:894", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:895", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 4, "node": "l:892", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 9, "node": "l:890", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:924", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:925", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "mlp", "layer": 4, "node": "l:922", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 8, "node": "l:920", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:948", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:949", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 4, "node": "l:946", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 11, "node": "l:944", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:972", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:973", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "mlp", "layer": 4, "node": "l:970", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 8, "node": "l:968", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:996", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:997", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 4, "node": "l:994", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 9, "node": "l:992", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:1020", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:1021", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 4, "node": "l:1018", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 11, "node": "l:1016", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:1044", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:1045", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:1042", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 5, "node": "l:1040", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "attn", "layer": 17, "node": "l:10", "routing": "attn" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 15, "node": "l:1057", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "attn", "layer": 17, "node": "l:0", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:863", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 4, "node": "l:837", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:789", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:741", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:692", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:620", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 9, "node": "l:210", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 8, "node": "l:608", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 7, "node": "l:596", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:569", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:557", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 12, "node": "l:524", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:454", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:86", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:1039", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 3, "node": "l:1034", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:1015", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 3, "node": "l:1010", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:991", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 3, "node": "l:986", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:967", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 3, "node": "l:962", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:943", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 4, "node": "l:938", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:911", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 0, "node": "l:908", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:829", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:834", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:816", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 5, "node": "l:811", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 2, "node": "l:781", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:786", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 3, "node": "l:768", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 3, "node": "l:763", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:732", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:738", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:719", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 5, "node": "l:714", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 2, "node": "l:683", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:675", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:657", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 5, "node": "l:654", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:638", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 9, "node": "l:149", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 9, "node": "l:241", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 9, "node": "l:207", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 4, "node": "l:96", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:232", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 4, "node": "l:199", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:581", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:542", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:536", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:509", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:496", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:490", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 14, "node": "l:475", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 14, "node": "l:457", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 13, "node": "l:439", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 14, "node": "l:421", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:414", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:401", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:384", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 13, "node": "l:377", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:361", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:343", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:337", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 13, "node": "l:319", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:306", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 8, "node": "l:303", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:283", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:271", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:250", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:221", "routing": "mlp" } ], "layers": [ { "active_attention_heads": 4, "active_mlp_neurons": 49, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L0.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:311", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 34, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L0.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:571", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 35, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 1 ], "id": "L0.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:610", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 36, 1 ] ], "type": "compute_linear" }, { "heads": [ 3, 1 ], "id": "L0.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": "l:865", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 37, 1 ] ], "type": "compute_linear" } ], "emitted_attention_heads": 4, "index": 0, "mlp_operations": [ { "hidden_slots": [ [ 1, 1 ] ], "id": "L0.mlp.0", "node": "l:44", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 38, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 2, 1 ] ], "id": "L0.mlp.1", "node": "l:46", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 39, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 1 ] ], "id": "L0.mlp.2", "node": "l:48", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 40, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 4, 1 ] ], "id": "L0.mlp.3", "node": "l:7", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 41, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 1 ] ], "id": "L0.mlp.4", "node": "l:112", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 42, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 6, 10 ] ], "id": "L0.mlp.5", "node": "l:121", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 43, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 16, 1 ] ], "id": "L0.mlp.6", "node": "l:26", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 44, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 17, 2 ] ], "id": "L0.mlp.7", "node": "l:310", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 34, 1 ] ], "source_columns_b": null, "target_columns": [ [ 45, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 19, 2 ] ], "id": "L0.mlp.8", "node": "l:575", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 1 ] ], "source_columns_b": null, "target_columns": [ [ 46, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 21, 2 ] ], "id": "L0.mlp.9", "node": "l:614", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 36, 1 ] ], "source_columns_b": null, "target_columns": [ [ 47, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 23, 2 ] ], "id": "L0.mlp.10", "node": "l:869", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 37, 1 ] ], "source_columns_b": null, "target_columns": [ [ 48, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 25, 2 ] ], "id": "L0.mlp.11", "node": "l:559", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 49, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 27, 2 ] ], "id": "L0.mlp.12", "node": "l:598", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 29, 2 ] ], "id": "L0.mlp.13", "node": "l:694", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 51, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 31, 2 ] ], "id": "L0.mlp.14", "node": "l:743", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 52, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 33, 2 ] ], "id": "L0.mlp.15", "node": "l:791", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 53, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 35, 2 ] ], "id": "L0.mlp.16", "node": "l:839", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 54, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 37, 6 ] ], "id": "L0.mlp.17", "node": "l:191", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 55, 3 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 43, 2 ] ], "id": "L0.mlp.18", "node": "l:358", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 58, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 45, 2 ] ], "id": "L0.mlp.19", "node": "l:242", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 59, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 47, 2 ] ], "id": "L0.mlp.20", "node": "l:260", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 60, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L0.mlp.21", "node": "l:109", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 61, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.22", "node": "l:110", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 62, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.23", "node": "l:113", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 63, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.24", "node": "l:162", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 64, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.25", "node": "l:163", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 65, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.26", "node": "l:106", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 66, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.27", "node": "l:125", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 67, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.28", "node": "l:126", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 68, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.29", "node": "l:128", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 69, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.30", "node": "l:23", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 70, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.31", "node": "l:24", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 71, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.32", "node": "l:27", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 72, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.33", "node": "l:137", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 73, 4 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.34", "node": "l:182", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 77, 4 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.35", "node": "l:102", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 81, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.36", "node": "l:155", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 82, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.37", "node": "l:20", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 83, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.38", "node": "l:134", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 84, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.39", "node": "l:140", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 85, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.40", "node": "l:179", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 118, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.41", "node": "l:195", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 119, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.42", "node": "l:351", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 120, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.43", "node": "l:392", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 121, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.44", "node": "l:429", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 122, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.45", "node": "l:465", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 123, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.46", "node": "l:681", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 124, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.47", "node": "l:730", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 125, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.48", "node": "l:779", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 126, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.49", "node": "l:827", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 127, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.50", "node": "l:503", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 128, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.51", "node": "l:673", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 129, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.52", "node": "l:735", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 130, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.53", "node": "l:784", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 131, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.54", "node": "l:832", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 132, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.55", "node": "l:566", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 133, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.56", "node": "l:605", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 134, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.57", "node": "l:578", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 135, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.58", "node": "l:617", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 136, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.59", "node": "l:682", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 137, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.60", "node": "l:731", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 138, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.61", "node": "l:828", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 139, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.62", "node": "l:701", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 140, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.63", "node": "l:750", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 141, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.64", "node": "l:798", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 142, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.65", "node": "l:846", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 143, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.66", "node": "l:872", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 144, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.67", "node": "l:660", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 145, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.68", "node": "l:814", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 146, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.69", "node": "l:914", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 147, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.70", "node": "l:989", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 148, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.71", "node": "l:1013", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 149, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.72", "node": "l:1037", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 150, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.73", "node": "l:436", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 151, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.74", "node": "l:472", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 152, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.75", "node": "l:381", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 153, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.76", "node": "l:418", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 154, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.77", "node": "l:443", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 155, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.78", "node": "l:479", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 156, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.79", "node": "l:513", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 157, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.80", "node": "l:546", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 158, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.81", "node": "l:585", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 159, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.82", "node": "l:340", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 160, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.83", "node": "l:37", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 161, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.84", "node": "l:56", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 162, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.85", "node": "l:63", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 163, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.86", "node": "l:65", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 164, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.87", "node": "l:72", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 165, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.88", "node": "l:74", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 166, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.89", "node": "l:908", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 167, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.90", "node": "l:632", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 168, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.91", "node": "l:857", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 201, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.92", "node": "l:66", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 234, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.93", "node": "l:635", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 267, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.94", "node": "l:75", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 300, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.95", "node": "l:889", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 333, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.96", "node": "l:1056", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 366, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.97", "node": "l:883", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 399, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.98", "node": "l:930", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 432, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.99", "node": "l:954", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 465, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.100", "node": "l:978", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 498, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.101", "node": "l:1002", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 531, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.102", "node": "l:1026", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 564, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.103", "node": "l:1050", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 597, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.104", "node": "l:886", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 630, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.105", "node": "l:903", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 663, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.106", "node": "l:933", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 696, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.107", "node": "l:981", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 729, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.108", "node": "l:1005", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 762, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.109", "node": "l:1053", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 795, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.110", "node": "l:4", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 828, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.111", "node": "l:5", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 829, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.112", "node": "l:9", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 830, 1 ] ], "type": "compute_literal_value" } ], "newly_computed": [ "l:26", "l:23", "l:24", "l:27", "l:22", "l:25", "l:20", "l:44", "l:46", "l:48", "l:7", "l:37", "l:56", "l:63", "l:65", "l:72", "l:74", "l:112", "l:125", "l:126", "l:128", "l:124", "l:121", "l:109", "l:110", "l:113", "l:108", "l:111", "l:106", "l:134", "l:137", "l:136", "l:140", "l:162", "l:163", "l:161", "l:179", "l:182", "l:102", "l:155", "l:154", "l:195", "l:242", "l:340", "l:381", "l:418", "l:559", "l:566", "l:598", "l:605", "l:660", "l:814", "l:632", "l:857", "l:191", "l:358", "l:311", "l:310", "l:351", "l:392", "l:429", "l:436", "l:465", "l:472", "l:503", "l:571", "l:578", "l:575", "l:610", "l:617", "l:614", "l:681", "l:682", "l:673", "l:730", "l:731", "l:735", "l:779", "l:784", "l:827", "l:828", "l:832", "l:66", "l:635", "l:260", "l:443", "l:479", "l:513", "l:546", "l:585", "l:694", "l:701", "l:743", "l:750", "l:791", "l:798", "l:839", "l:846", "l:865", "l:872", "l:869", "l:914", "l:989", "l:1013", "l:1037", "l:75", "l:889", "l:1056", "l:883", "l:930", "l:954", "l:978", "l:1002", "l:1026", "l:1050", "l:886", "l:903", "l:933", "l:981", "l:1005", "l:1053", "l:4", "l:5", "l:9", "l:3", "l:6", "l:908" ] }, { "active_attention_heads": 32, "active_mlp_neurons": 156, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L1.attn.0", "key_source_columns": [ [ 42, 1 ], [ 63, 1 ] ], "node": "l:107", "query_source_columns": [ [ 61, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 42, 1 ] ], "source_columns_b": null, "target_columns": [ [ 831, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L1.attn.1", "key_source_columns": [ [ 44, 1 ], [ 72, 1 ] ], "node": "l:21", "query_source_columns": [ [ 70, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 44, 1 ] ], "source_columns_b": null, "target_columns": [ [ 832, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L1.attn.2", "key_source_columns": [ [ 120, 1 ] ], "node": "l:350", "query_source_columns": [ [ 120, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 833, 33 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L1.attn.3", "key_source_columns": [ [ 121, 1 ] ], "node": "l:391", "query_source_columns": [ [ 121, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 866, 33 ] ], "type": "compute_attn" }, { "heads": [ 4, 1 ], "id": "L1.attn.4", "key_source_columns": [ [ 122, 1 ] ], "node": "l:428", "query_source_columns": [ [ 122, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 899, 33 ] ], "type": "compute_attn" }, { "heads": [ 5, 1 ], "id": "L1.attn.5", "key_source_columns": [ [ 123, 1 ] ], "node": "l:464", "query_source_columns": [ [ 123, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 932, 33 ] ], "type": "compute_attn" }, { "heads": [ 6, 1 ], "id": "L1.attn.6", "key_source_columns": [ [ 125, 1 ] ], "node": "l:729", "query_source_columns": [ [ 125, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 965, 33 ] ], "type": "compute_attn" }, { "heads": [ 7, 1 ], "id": "L1.attn.7", "key_source_columns": [ [ 126, 1 ] ], "node": "l:778", "query_source_columns": [ [ 126, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 998, 33 ] ], "type": "compute_attn" }, { "heads": [ 8, 1 ], "id": "L1.attn.8", "key_source_columns": [ [ 127, 1 ] ], "node": "l:826", "query_source_columns": [ [ 127, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1031, 33 ] ], "type": "compute_attn" }, { "heads": [ 9, 1 ], "id": "L1.attn.9", "key_source_columns": [ [ 128, 1 ] ], "node": "l:502", "query_source_columns": [ [ 128, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1064, 33 ] ], "type": "compute_attn" }, { "heads": [ 10, 1 ], "id": "L1.attn.10", "key_source_columns": [ [ 129, 1 ] ], "node": "l:672", "query_source_columns": [ [ 129, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1097, 33 ] ], "type": "compute_attn" }, { "heads": [ 11, 1 ], "id": "L1.attn.11", "key_source_columns": [ [ 131, 1 ] ], "node": "l:783", "query_source_columns": [ [ 131, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1130, 33 ] ], "type": "compute_attn" }, { "heads": [ 12, 1 ], "id": "L1.attn.12", "key_source_columns": [ [ 132, 1 ] ], "node": "l:831", "query_source_columns": [ [ 132, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1163, 33 ] ], "type": "compute_attn" }, { "heads": [ 13, 1 ], "id": "L1.attn.13", "key_source_columns": [ [ 133, 1 ] ], "node": "l:565", "query_source_columns": [ [ 133, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1196, 33 ] ], "type": "compute_attn" }, { "heads": [ 14, 1 ], "id": "L1.attn.14", "key_source_columns": [ [ 134, 1 ] ], "node": "l:604", "query_source_columns": [ [ 134, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1229, 33 ] ], "type": "compute_attn" }, { "heads": [ 15, 1 ], "id": "L1.attn.15", "key_source_columns": [ [ 135, 1 ] ], "node": "l:577", "query_source_columns": [ [ 135, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1262, 33 ] ], "type": "compute_attn" }, { "heads": [ 16, 1 ], "id": "L1.attn.16", "key_source_columns": [ [ 136, 1 ] ], "node": "l:616", "query_source_columns": [ [ 136, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1295, 33 ] ], "type": "compute_attn" }, { "heads": [ 17, 1 ], "id": "L1.attn.17", "key_source_columns": [ [ 140, 1 ] ], "node": "l:700", "query_source_columns": [ [ 140, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1328, 33 ] ], "type": "compute_attn" }, { "heads": [ 18, 1 ], "id": "L1.attn.18", "key_source_columns": [ [ 141, 1 ] ], "node": "l:749", "query_source_columns": [ [ 141, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1361, 33 ] ], "type": "compute_attn" }, { "heads": [ 19, 1 ], "id": "L1.attn.19", "key_source_columns": [ [ 142, 1 ] ], "node": "l:797", "query_source_columns": [ [ 142, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1394, 33 ] ], "type": "compute_attn" }, { "heads": [ 20, 1 ], "id": "L1.attn.20", "key_source_columns": [ [ 143, 1 ] ], "node": "l:845", "query_source_columns": [ [ 143, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1427, 33 ] ], "type": "compute_attn" }, { "heads": [ 21, 1 ], "id": "L1.attn.21", "key_source_columns": [ [ 144, 1 ] ], "node": "l:871", "query_source_columns": [ [ 144, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1460, 33 ] ], "type": "compute_attn" }, { "heads": [ 22, 1 ], "id": "L1.attn.22", "key_source_columns": [ [ 151, 1 ] ], "node": "l:435", "query_source_columns": [ [ 151, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1493, 33 ] ], "type": "compute_attn" }, { "heads": [ 23, 1 ], "id": "L1.attn.23", "key_source_columns": [ [ 152, 1 ] ], "node": "l:471", "query_source_columns": [ [ 152, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1526, 33 ] ], "type": "compute_attn" }, { "heads": [ 24, 1 ], "id": "L1.attn.24", "key_source_columns": [ [ 153, 1 ] ], "node": "l:380", "query_source_columns": [ [ 153, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1559, 33 ] ], "type": "compute_attn" }, { "heads": [ 25, 1 ], "id": "L1.attn.25", "key_source_columns": [ [ 154, 1 ] ], "node": "l:417", "query_source_columns": [ [ 154, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1592, 33 ] ], "type": "compute_attn" }, { "heads": [ 26, 1 ], "id": "L1.attn.26", "key_source_columns": [ [ 155, 1 ] ], "node": "l:442", "query_source_columns": [ [ 155, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1625, 33 ] ], "type": "compute_attn" }, { "heads": [ 27, 1 ], "id": "L1.attn.27", "key_source_columns": [ [ 156, 1 ] ], "node": "l:478", "query_source_columns": [ [ 156, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1658, 33 ] ], "type": "compute_attn" }, { "heads": [ 28, 1 ], "id": "L1.attn.28", "key_source_columns": [ [ 157, 1 ] ], "node": "l:512", "query_source_columns": [ [ 157, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1691, 33 ] ], "type": "compute_attn" }, { "heads": [ 29, 1 ], "id": "L1.attn.29", "key_source_columns": [ [ 158, 1 ] ], "node": "l:545", "query_source_columns": [ [ 158, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1724, 33 ] ], "type": "compute_attn" }, { "heads": [ 30, 1 ], "id": "L1.attn.30", "key_source_columns": [ [ 159, 1 ] ], "node": "l:584", "query_source_columns": [ [ 159, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1757, 33 ] ], "type": "compute_attn" }, { "heads": [ 31, 1 ], "id": "L1.attn.31", "key_source_columns": [ [ 41, 1 ], [ 830, 1 ] ], "node": "l:2", "query_source_columns": [ [ 828, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 41, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1790, 1 ] ], "type": "compute_attn" } ], "emitted_attention_heads": 32, "index": 1, "mlp_operations": [ { "hidden_slots": [ [ 1, 1 ] ], "id": "L1.mlp.0", "node": "l:564", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1196, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1791, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 2, 1 ] ], "id": "L1.mlp.1", "node": "l:603", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1229, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1792, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 1 ] ], "id": "L1.mlp.2", "node": "l:576", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1262, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1793, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 4, 1 ] ], "id": "L1.mlp.3", "node": "l:615", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1295, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1794, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 1 ] ], "id": "L1.mlp.4", "node": "l:699", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1328, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1795, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 6, 1 ] ], "id": "L1.mlp.5", "node": "l:748", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1361, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1796, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 7, 1 ] ], "id": "L1.mlp.6", "node": "l:796", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1394, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1797, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 8, 1 ] ], "id": "L1.mlp.7", "node": "l:844", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1427, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1798, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 9, 1 ] ], "id": "L1.mlp.8", "node": "l:870", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1460, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1799, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 10, 4 ] ], "id": "L1.mlp.9", "node": "l:135", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 42, 1 ], [ 73, 4 ] ], "source_columns_b": null, "target_columns": [ [ 1800, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 14, 2 ] ], "id": "L1.mlp.10", "node": "l:43", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 38, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1804, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 16, 2 ] ], "id": "L1.mlp.11", "node": "l:45", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 39, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1805, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 18, 2 ] ], "id": "L1.mlp.12", "node": "l:47", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 40, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1806, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 20, 2 ] ], "id": "L1.mlp.13", "node": "l:563", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 49, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1807, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 22, 2 ] ], "id": "L1.mlp.14", "node": "l:602", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 50, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1808, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 24, 2 ] ], "id": "L1.mlp.15", "node": "l:698", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 51, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1809, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 26, 2 ] ], "id": "L1.mlp.16", "node": "l:747", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 52, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1810, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 28, 2 ] ], "id": "L1.mlp.17", "node": "l:795", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 53, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1811, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 30, 2 ] ], "id": "L1.mlp.18", "node": "l:843", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 54, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1812, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 32, 2 ] ], "id": "L1.mlp.19", "node": "l:349", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 833, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1813, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 34, 2 ] ], "id": "L1.mlp.20", "node": "l:390", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 866, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1814, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 36, 2 ] ], "id": "L1.mlp.21", "node": "l:427", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 899, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1815, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 38, 2 ] ], "id": "L1.mlp.22", "node": "l:463", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 932, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1816, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 40, 2 ] ], "id": "L1.mlp.23", "node": "l:825", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1031, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1817, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 42, 2 ] ], "id": "L1.mlp.24", "node": "l:501", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1064, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1818, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 44, 2 ] ], "id": "L1.mlp.25", "node": "l:671", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1097, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1819, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 46, 2 ] ], "id": "L1.mlp.26", "node": "l:782", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1130, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1820, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 48, 2 ] ], "id": "L1.mlp.27", "node": "l:830", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1163, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1821, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 50, 2 ] ], "id": "L1.mlp.28", "node": "l:434", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1493, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1822, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 52, 2 ] ], "id": "L1.mlp.29", "node": "l:470", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1526, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1823, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 54, 2 ] ], "id": "L1.mlp.30", "node": "l:379", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1559, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1824, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 56, 2 ] ], "id": "L1.mlp.31", "node": "l:416", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1592, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1825, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 58, 2 ] ], "id": "L1.mlp.32", "node": "l:441", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1625, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1826, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 60, 2 ] ], "id": "L1.mlp.33", "node": "l:477", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1658, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1827, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 62, 2 ] ], "id": "L1.mlp.34", "node": "l:511", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1691, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1828, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 64, 2 ] ], "id": "L1.mlp.35", "node": "l:544", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1724, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1829, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 66, 2 ] ], "id": "L1.mlp.36", "node": "l:583", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1757, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1830, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 68, 2 ] ], "id": "L1.mlp.37", "node": "l:114", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 42, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1831, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 70, 2 ] ], "id": "L1.mlp.38", "node": "l:28", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 44, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1832, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 72, 2 ] ], "id": "L1.mlp.39", "node": "l:357", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 58, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1833, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 74, 2 ] ], "id": "L1.mlp.40", "node": "l:398", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 58, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1834, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L1.mlp.41", "node": "l:52", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1835, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L1.mlp.42", "node": "l:53", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1836, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L1.mlp.43", "node": "l:55", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1837, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L1.mlp.44", "node": "l:780", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1838, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L1.mlp.45", "node": "l:717", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1839, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L1.mlp.46", "node": "l:766", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1840, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L1.mlp.47", "node": "l:941", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1841, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L1.mlp.48", "node": "l:965", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1842, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L1.mlp.49", "node": "l:493", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1843, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L1.mlp.50", "node": "l:641", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1844, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 76, 2 ] ], "id": "L1.mlp.51", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 70, 1 ] ], "source_columns_b": null, "target_columns": [ [ 70, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 78, 2 ] ], "id": "L1.mlp.52", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 71, 1 ] ], "source_columns_b": null, "target_columns": [ [ 71, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 80, 2 ] ], "id": "L1.mlp.53", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 72, 1 ] ], "source_columns_b": null, "target_columns": [ [ 72, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 82, 2 ] ], "id": "L1.mlp.54", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 42, 1 ] ], "source_columns_b": null, "target_columns": [ [ 42, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 84, 2 ] ], "id": "L1.mlp.55", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 61, 1 ] ], "source_columns_b": null, "target_columns": [ [ 61, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 86, 2 ] ], "id": "L1.mlp.56", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 62, 1 ] ], "source_columns_b": null, "target_columns": [ [ 62, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 88, 2 ] ], "id": "L1.mlp.57", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 63, 1 ] ], "source_columns_b": null, "target_columns": [ [ 63, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 90, 2 ] ], "id": "L1.mlp.58", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 153, 1 ] ], "source_columns_b": null, "target_columns": [ [ 153, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 92, 2 ] ], "id": "L1.mlp.59", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 154, 1 ] ], "source_columns_b": null, "target_columns": [ [ 154, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 94, 2 ] ], "id": "L1.mlp.60", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 133, 1 ] ], "source_columns_b": null, "target_columns": [ [ 133, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 96, 2 ] ], "id": "L1.mlp.61", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 134, 1 ] ], "source_columns_b": null, "target_columns": [ [ 134, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 98, 2 ] ], "id": "L1.mlp.62", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 34, 1 ] ], "source_columns_b": null, "target_columns": [ [ 34, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 100, 2 ] ], "id": "L1.mlp.63", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 120, 1 ] ], "source_columns_b": null, "target_columns": [ [ 120, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 102, 2 ] ], "id": "L1.mlp.64", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 121, 1 ] ], "source_columns_b": null, "target_columns": [ [ 121, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 104, 2 ] ], "id": "L1.mlp.65", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 122, 1 ] ], "source_columns_b": null, "target_columns": [ [ 122, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 106, 2 ] ], "id": "L1.mlp.66", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 151, 1 ] ], "source_columns_b": null, "target_columns": [ [ 151, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 108, 2 ] ], "id": "L1.mlp.67", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 123, 1 ] ], "source_columns_b": null, "target_columns": [ [ 123, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 110, 2 ] ], "id": "L1.mlp.68", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 152, 1 ] ], "source_columns_b": null, "target_columns": [ [ 152, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 112, 2 ] ], "id": "L1.mlp.69", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 128, 1 ] ], "source_columns_b": null, "target_columns": [ [ 128, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 114, 2 ] ], "id": "L1.mlp.70", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 135, 1 ] ], "source_columns_b": null, "target_columns": [ [ 135, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 116, 2 ] ], "id": "L1.mlp.71", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 136, 1 ] ], "source_columns_b": null, "target_columns": [ [ 136, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 118, 2 ] ], "id": "L1.mlp.72", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 129, 1 ] ], "source_columns_b": null, "target_columns": [ [ 129, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 120, 2 ] ], "id": "L1.mlp.73", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 125, 1 ] ], "source_columns_b": null, "target_columns": [ [ 125, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 122, 2 ] ], "id": "L1.mlp.74", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 126, 1 ] ], "source_columns_b": null, "target_columns": [ [ 126, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 124, 2 ] ], "id": "L1.mlp.75", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 131, 1 ] ], "source_columns_b": null, "target_columns": [ [ 131, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 126, 2 ] ], "id": "L1.mlp.76", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 127, 1 ] ], "source_columns_b": null, "target_columns": [ [ 127, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 128, 2 ] ], "id": "L1.mlp.77", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 132, 1 ] ], "source_columns_b": null, "target_columns": [ [ 132, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 130, 2 ] ], "id": "L1.mlp.78", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 155, 1 ] ], "source_columns_b": null, "target_columns": [ [ 155, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 132, 2 ] ], "id": "L1.mlp.79", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 156, 1 ] ], "source_columns_b": null, "target_columns": [ [ 156, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 134, 2 ] ], "id": "L1.mlp.80", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 157, 1 ] ], "source_columns_b": null, "target_columns": [ [ 157, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 136, 2 ] ], "id": "L1.mlp.81", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 158, 1 ] ], "source_columns_b": null, "target_columns": [ [ 158, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 138, 2 ] ], "id": "L1.mlp.82", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 159, 1 ] ], "source_columns_b": null, "target_columns": [ [ 159, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 140, 2 ] ], "id": "L1.mlp.83", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 140, 1 ] ], "source_columns_b": null, "target_columns": [ [ 140, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 142, 2 ] ], "id": "L1.mlp.84", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 141, 1 ] ], "source_columns_b": null, "target_columns": [ [ 141, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 144, 2 ] ], "id": "L1.mlp.85", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 142, 1 ] ], "source_columns_b": null, "target_columns": [ [ 142, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 146, 2 ] ], "id": "L1.mlp.86", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 143, 1 ] ], "source_columns_b": null, "target_columns": [ [ 143, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 148, 2 ] ], "id": "L1.mlp.87", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 144, 1 ] ], "source_columns_b": null, "target_columns": [ [ 144, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 150, 2 ] ], "id": "L1.mlp.88", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 828, 1 ] ], "source_columns_b": null, "target_columns": [ [ 828, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 152, 2 ] ], "id": "L1.mlp.89", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 829, 1 ] ], "source_columns_b": null, "target_columns": [ [ 829, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 154, 2 ] ], "id": "L1.mlp.90", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 830, 1 ] ], "source_columns_b": null, "target_columns": [ [ 830, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:28", "l:21", "l:43", "l:45", "l:47", "l:42", "l:52", "l:53", "l:55", "l:51", "l:54", "l:107", "l:114", "l:135", "l:380", "l:379", "l:417", "l:416", "l:493", "l:565", "l:564", "l:563", "l:562", "l:604", "l:603", "l:602", "l:601", "l:717", "l:766", "l:350", "l:349", "l:357", "l:391", "l:390", "l:398", "l:428", "l:427", "l:435", "l:434", "l:464", "l:463", "l:471", "l:470", "l:502", "l:501", "l:577", "l:576", "l:574", "l:616", "l:615", "l:613", "l:672", "l:671", "l:729", "l:778", "l:780", "l:783", "l:782", "l:826", "l:825", "l:824", "l:831", "l:830", "l:442", "l:441", "l:478", "l:477", "l:512", "l:511", "l:545", "l:544", "l:584", "l:583", "l:641", "l:700", "l:699", "l:698", "l:697", "l:749", "l:748", "l:747", "l:746", "l:797", "l:796", "l:795", "l:794", "l:845", "l:844", "l:843", "l:842", "l:871", "l:870", "l:868", "l:941", "l:965", "l:2" ] }, { "active_attention_heads": 32, "active_mlp_neurons": 189, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L2.attn.0", "key_source_columns": [ [ 41, 1 ], [ 1837, 1 ] ], "node": "l:50", "query_source_columns": [ [ 1835, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 41, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1662, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L2.attn.1", "key_source_columns": [ [ 831, 1 ] ], "node": "l:105", "query_source_columns": [ [ 66, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1831, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1663, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L2.attn.2", "key_source_columns": [ [ 832, 1 ] ], "node": "l:19", "query_source_columns": [ [ 83, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1832, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1635, 1 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L2.attn.3", "key_source_columns": [ [ 124, 1 ] ], "node": "l:680", "query_source_columns": [ [ 124, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1636, 3 ], [ 1640, 1 ], [ 1643, 1 ], [ 1664, 17 ], [ 1688, 6 ], [ 1695, 2 ], [ 1702, 2 ], [ 1706, 1 ] ], "type": "compute_attn" }, { "heads": [ 4, 1 ], "id": "L2.attn.4", "key_source_columns": [ [ 130, 1 ] ], "node": "l:734", "query_source_columns": [ [ 130, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 83, 1 ], [ 1644, 5 ], [ 1694, 1 ], [ 1697, 1 ], [ 1708, 25 ] ], "type": "compute_attn" }, { "heads": [ 5, 1 ], "id": "L2.attn.5", "key_source_columns": [ [ 1840, 1 ] ], "node": "l:765", "query_source_columns": [ [ 1840, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 120, 5 ], [ 1642, 1 ], [ 1649, 5 ], [ 1698, 1 ], [ 1707, 1 ], [ 1733, 20 ] ], "type": "compute_attn" }, { "heads": [ 6, 1 ], "id": "L2.attn.6", "key_source_columns": [ [ 1842, 1 ] ], "node": "l:964", "query_source_columns": [ [ 1842, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 66, 1 ], [ 70, 1 ], [ 125, 12 ], [ 1654, 3 ], [ 1753, 16 ] ], "type": "compute_attn" }, { "heads": [ 7, 1 ], "id": "L2.attn.7", "key_source_columns": [ [ 148, 1 ] ], "node": "l:988", "query_source_columns": [ [ 148, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 71, 1 ], [ 140, 5 ], [ 148, 8 ], [ 1657, 3 ], [ 1699, 1 ], [ 1769, 15 ] ], "type": "compute_attn" }, { "heads": [ 8, 1 ], "id": "L2.attn.8", "key_source_columns": [ [ 149, 1 ] ], "node": "l:1012", "query_source_columns": [ [ 149, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 156, 5 ], [ 828, 20 ], [ 1660, 2 ], [ 1784, 6 ] ], "type": "compute_attn" }, { "heads": [ 9, 1 ], "id": "L2.attn.9", "key_source_columns": [ [ 150, 1 ] ], "node": "l:1036", "query_source_columns": [ [ 150, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 34, 1 ], [ 848, 13 ], [ 862, 19 ] ], "type": "compute_attn" }, { "heads": [ 10, 1 ], "id": "L2.attn.10", "key_source_columns": [ [ 160, 1 ] ], "node": "l:339", "query_source_columns": [ [ 160, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 72, 1 ], [ 881, 8 ], [ 890, 24 ] ], "type": "compute_attn" }, { "heads": [ 11, 1 ], "id": "L2.attn.11", "key_source_columns": [ [ 1843, 1 ] ], "node": "l:492", "query_source_columns": [ [ 1843, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 914, 33 ] ], "type": "compute_attn" }, { "heads": [ 12, 1 ], "id": "L2.attn.12", "key_source_columns": [ [ 1844, 1 ] ], "node": "l:640", "query_source_columns": [ [ 1844, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 947, 33 ] ], "type": "compute_attn" }, { "heads": [ 13, 1 ], "id": "L2.attn.13", "key_source_columns": [ [ 0, 1 ] ], "node": "l:728", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 965, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1823, 1 ] ], "type": "compute_linear" }, { "heads": [ 14, 1 ], "id": "L2.attn.14", "key_source_columns": [ [ 0, 1 ] ], "node": "l:777", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 998, 33 ] ], "source_columns_b": null, "target_columns": [ [ 981, 1 ] ], "type": "compute_linear" }, { "heads": [ 15, 1 ], "id": "L2.attn.15", "key_source_columns": [ [ 0, 1 ] ], "node": "l:469", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1823, 1 ] ], "source_columns_b": null, "target_columns": [ [ 982, 1 ] ], "type": "compute_linear" }, { "heads": [ 16, 16 ], "id": "L2.attn.16", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 44, 1 ], [ 58, 1 ], [ 66, 1 ], [ 73, 4 ], [ 83, 1 ], [ 124, 1 ], [ 130, 1 ], [ 148, 3 ], [ 160, 1 ], [ 831, 959 ], [ 1823, 1 ], [ 1831, 2 ], [ 1835, 3 ], [ 1840, 1 ], [ 1842, 3 ] ], "type": "cancel" } ], "emitted_attention_heads": 32, "index": 2, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L2.mlp.0", "node": "l:41", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1804, 3 ] ], "source_columns_b": null, "target_columns": [ [ 983, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L2.mlp.1", "node": "l:49", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1662, 1 ] ], "source_columns_b": null, "target_columns": [ [ 984, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 32 ] ], "id": "L2.mlp.2", "node": "l:104", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1663, 1 ] ], "source_columns_b": null, "target_columns": [ [ 985, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 37, 2 ] ], "id": "L2.mlp.3", "node": "l:173", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1662, 1 ] ], "source_columns_b": null, "target_columns": [ [ 986, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 39, 45 ] ], "id": "L2.mlp.4", "node": "l:18", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1635, 1 ] ], "source_columns_b": null, "target_columns": [ [ 987, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 84, 2 ] ], "id": "L2.mlp.5", "node": "l:348", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1813, 1 ] ], "source_columns_b": null, "target_columns": [ [ 988, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 86, 2 ] ], "id": "L2.mlp.6", "node": "l:389", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1814, 1 ] ], "source_columns_b": null, "target_columns": [ [ 989, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 88, 2 ] ], "id": "L2.mlp.7", "node": "l:426", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1815, 1 ] ], "source_columns_b": null, "target_columns": [ [ 990, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 90, 2 ] ], "id": "L2.mlp.8", "node": "l:462", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1816, 1 ] ], "source_columns_b": null, "target_columns": [ [ 991, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 92, 2 ] ], "id": "L2.mlp.9", "node": "l:500", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1818, 1 ] ], "source_columns_b": null, "target_columns": [ [ 992, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 94, 3 ] ], "id": "L2.mlp.10", "node": "l:726", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1823, 1 ], [ 138, 1 ] ], "source_columns_b": null, "target_columns": [ [ 993, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 97, 3 ] ], "id": "L2.mlp.11", "node": "l:775", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 981, 1 ], [ 1838, 1 ] ], "source_columns_b": null, "target_columns": [ [ 994, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 100, 3 ] ], "id": "L2.mlp.12", "node": "l:823", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1817, 1 ], [ 139, 1 ] ], "source_columns_b": null, "target_columns": [ [ 995, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 103, 6 ] ], "id": "L2.mlp.13", "node": "l:829", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1821, 1 ] ], "source_columns_b": null, "target_columns": [ [ 996, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 109, 6 ] ], "id": "L2.mlp.14", "node": "l:781", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1820, 1 ] ], "source_columns_b": null, "target_columns": [ [ 997, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 115, 6 ] ], "id": "L2.mlp.15", "node": "l:683", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1819, 1 ] ], "source_columns_b": null, "target_columns": [ [ 998, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 121, 2 ] ], "id": "L2.mlp.16", "node": "l:561", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1807, 1 ], [ 1791, 1 ] ], "source_columns_b": null, "target_columns": [ [ 999, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 123, 2 ] ], "id": "L2.mlp.17", "node": "l:600", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1808, 1 ], [ 1792, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1000, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 125, 2 ] ], "id": "L2.mlp.18", "node": "l:573", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 46, 1 ], [ 1793, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1001, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 127, 2 ] ], "id": "L2.mlp.19", "node": "l:612", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 47, 1 ], [ 1794, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1002, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 129, 2 ] ], "id": "L2.mlp.20", "node": "l:696", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1809, 1 ], [ 1795, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1003, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 131, 2 ] ], "id": "L2.mlp.21", "node": "l:745", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1810, 1 ], [ 1796, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1004, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 133, 2 ] ], "id": "L2.mlp.22", "node": "l:793", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1811, 1 ], [ 1797, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1005, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 135, 2 ] ], "id": "L2.mlp.23", "node": "l:841", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1812, 1 ], [ 1798, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1006, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 137, 2 ] ], "id": "L2.mlp.24", "node": "l:867", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 48, 1 ], [ 1799, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1007, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 139, 2 ] ], "id": "L2.mlp.25", "node": "l:670", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1819, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1008, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 141, 1 ] ], "id": "L2.mlp.26", "node": "l:491", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 914, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1009, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 142, 1 ] ], "id": "L2.mlp.27", "node": "l:639", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 947, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1010, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 143, 2 ] ], "id": "L2.mlp.28", "node": "l:733", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 83, 1 ], [ 1644, 5 ], [ 1694, 1 ], [ 1697, 1 ], [ 1708, 25 ] ], "source_columns_b": null, "target_columns": [ [ 1011, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 145, 2 ] ], "id": "L2.mlp.29", "node": "l:764", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 120, 5 ], [ 1642, 1 ], [ 1649, 5 ], [ 1698, 1 ], [ 1707, 1 ], [ 1733, 20 ] ], "source_columns_b": null, "target_columns": [ [ 1012, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 147, 2 ] ], "id": "L2.mlp.30", "node": "l:963", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 66, 1 ], [ 70, 1 ], [ 125, 12 ], [ 1654, 3 ], [ 1753, 16 ] ], "source_columns_b": null, "target_columns": [ [ 1013, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 149, 2 ] ], "id": "L2.mlp.31", "node": "l:987", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 71, 1 ], [ 140, 5 ], [ 148, 8 ], [ 1657, 3 ], [ 1699, 1 ], [ 1769, 15 ] ], "source_columns_b": null, "target_columns": [ [ 73, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 151, 2 ] ], "id": "L2.mlp.32", "node": "l:1011", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 156, 5 ], [ 828, 20 ], [ 1660, 2 ], [ 1784, 6 ] ], "source_columns_b": null, "target_columns": [ [ 1015, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 153, 2 ] ], "id": "L2.mlp.33", "node": "l:1035", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 34, 1 ], [ 848, 13 ], [ 862, 19 ] ], "source_columns_b": null, "target_columns": [ [ 1016, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 155, 2 ] ], "id": "L2.mlp.34", "node": "l:338", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 72, 1 ], [ 881, 8 ], [ 890, 24 ] ], "source_columns_b": null, "target_columns": [ [ 1017, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 157, 2 ] ], "id": "L2.mlp.35", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1805, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1805, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 159, 2 ] ], "id": "L2.mlp.36", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1807, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1807, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 161, 2 ] ], "id": "L2.mlp.37", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1813, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1813, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 163, 2 ] ], "id": "L2.mlp.38", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1814, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1814, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 165, 2 ] ], "id": "L2.mlp.39", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1815, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1815, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 167, 2 ] ], "id": "L2.mlp.40", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1816, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1816, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 169, 2 ] ], "id": "L2.mlp.41", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1793, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1793, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 171, 2 ] ], "id": "L2.mlp.42", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 47, 1 ] ], "source_columns_b": null, "target_columns": [ [ 47, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 173, 2 ] ], "id": "L2.mlp.43", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1820, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1820, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 175, 2 ] ], "id": "L2.mlp.44", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 139, 1 ] ], "source_columns_b": null, "target_columns": [ [ 139, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 177, 2 ] ], "id": "L2.mlp.45", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1821, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1821, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 179, 2 ] ], "id": "L2.mlp.46", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1809, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1809, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 181, 2 ] ], "id": "L2.mlp.47", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1811, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1811, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 183, 2 ] ], "id": "L2.mlp.48", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1798, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1798, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 185, 2 ] ], "id": "L2.mlp.49", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1799, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1799, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 187, 2 ] ], "id": "L2.mlp.50", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 48, 1 ] ], "source_columns_b": null, "target_columns": [ [ 48, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:19", "l:18", "l:41", "l:50", "l:49", "l:40", "l:173", "l:105", "l:104", "l:339", "l:338", "l:492", "l:491", "l:561", "l:600", "l:765", "l:764", "l:348", "l:389", "l:426", "l:462", "l:469", "l:500", "l:573", "l:612", "l:680", "l:670", "l:728", "l:727", "l:726", "l:734", "l:733", "l:777", "l:776", "l:775", "l:823", "l:640", "l:639", "l:696", "l:745", "l:793", "l:841", "l:867", "l:964", "l:963", "l:988", "l:987", "l:1012", "l:1011", "l:1036", "l:1035", "l:829", "l:781", "l:683" ] }, { "active_attention_heads": 22, "active_mlp_neurons": 314, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L3.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:774", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 997, 1 ] ], "source_columns_b": null, "target_columns": [ [ 994, 1 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L3.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:822", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 996, 1 ] ], "source_columns_b": null, "target_columns": [ [ 995, 1 ] ], "type": "add_into" }, { "heads": [ 2, 1 ], "id": "L3.attn.2", "key_source_columns": [ [ 145, 1 ] ], "node": "l:659", "query_source_columns": [ [ 145, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 46, 3 ], [ 83, 1 ], [ 1644, 5 ], [ 1694, 1 ], [ 1697, 1 ], [ 1708, 22 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L3.attn.3", "key_source_columns": [ [ 1839, 1 ] ], "node": "l:716", "query_source_columns": [ [ 1839, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 120, 4 ], [ 1642, 1 ], [ 1649, 4 ], [ 1698, 1 ], [ 1707, 1 ], [ 1730, 22 ] ], "type": "compute_attn" }, { "heads": [ 4, 1 ], "id": "L3.attn.4", "key_source_columns": [ [ 146, 1 ] ], "node": "l:813", "query_source_columns": [ [ 146, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 70, 1 ], [ 124, 13 ], [ 138, 1 ], [ 1653, 3 ], [ 1752, 15 ] ], "type": "compute_attn" }, { "heads": [ 5, 1 ], "id": "L3.attn.5", "key_source_columns": [ [ 147, 1 ] ], "node": "l:913", "query_source_columns": [ [ 147, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 66, 1 ], [ 71, 1 ], [ 139, 13 ], [ 1656, 3 ], [ 1699, 1 ], [ 1767, 14 ] ], "type": "compute_attn" }, { "heads": [ 6, 1 ], "id": "L3.attn.6", "key_source_columns": [ [ 1841, 1 ] ], "node": "l:940", "query_source_columns": [ [ 1841, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 152, 9 ], [ 828, 8 ], [ 1659, 3 ], [ 1781, 9 ], [ 1791, 4 ] ], "type": "compute_attn" }, { "heads": [ 7, 1 ], "id": "L3.attn.7", "key_source_columns": [ [ 0, 1 ] ], "node": "l:17", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 987, 1 ] ], "source_columns_b": null, "target_columns": [ [ 836, 1 ] ], "type": "compute_linear" }, { "heads": [ 8, 1 ], "id": "L3.attn.8", "key_source_columns": [ [ 0, 1 ] ], "node": "l:560", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 999, 1 ] ], "source_columns_b": null, "target_columns": [ [ 837, 1 ] ], "type": "compute_linear" }, { "heads": [ 9, 1 ], "id": "L3.attn.9", "key_source_columns": [ [ 0, 1 ] ], "node": "l:599", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1000, 1 ] ], "source_columns_b": null, "target_columns": [ [ 838, 1 ] ], "type": "compute_linear" }, { "heads": [ 10, 1 ], "id": "L3.attn.10", "key_source_columns": [ [ 0, 1 ] ], "node": "l:572", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1001, 1 ] ], "source_columns_b": null, "target_columns": [ [ 839, 1 ] ], "type": "compute_linear" }, { "heads": [ 11, 1 ], "id": "L3.attn.11", "key_source_columns": [ [ 0, 1 ] ], "node": "l:611", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1002, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1795, 1 ] ], "type": "compute_linear" }, { "heads": [ 12, 1 ], "id": "L3.attn.12", "key_source_columns": [ [ 0, 1 ] ], "node": "l:744", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1004, 1 ] ], "source_columns_b": null, "target_columns": [ [ 841, 1 ] ], "type": "compute_linear" }, { "heads": [ 13, 1 ], "id": "L3.attn.13", "key_source_columns": [ [ 0, 1 ] ], "node": "l:792", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1005, 1 ] ], "source_columns_b": null, "target_columns": [ [ 842, 1 ] ], "type": "compute_linear" }, { "heads": [ 14, 1 ], "id": "L3.attn.14", "key_source_columns": [ [ 0, 1 ] ], "node": "l:840", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1006, 1 ] ], "source_columns_b": null, "target_columns": [ [ 843, 1 ] ], "type": "compute_linear" }, { "heads": [ 15, 1 ], "id": "L3.attn.15", "key_source_columns": [ [ 0, 1 ] ], "node": "l:866", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1007, 1 ] ], "source_columns_b": null, "target_columns": [ [ 844, 1 ] ], "type": "compute_linear" }, { "heads": [ 16, 6 ], "id": "L3.attn.16", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 1 ], [ 46, 1 ], [ 66, 1 ], [ 70, 3 ], [ 83, 1 ], [ 120, 17 ], [ 138, 1 ], [ 140, 21 ], [ 828, 33 ], [ 862, 27 ], [ 890, 90 ], [ 981, 1 ], [ 987, 1 ], [ 999, 4 ], [ 1004, 4 ], [ 1635, 1 ], [ 1642, 1 ], [ 1644, 20 ], [ 1694, 1 ], [ 1697, 3 ], [ 1707, 83 ], [ 1791, 2 ], [ 1794, 4 ], [ 1804, 1 ], [ 1806, 1 ], [ 1808, 1 ], [ 1810, 1 ], [ 1812, 1 ], [ 1817, 3 ], [ 1823, 1 ], [ 1838, 2 ], [ 1841, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 22, "index": 3, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L3.mlp.0", "node": "l:39", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 983, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1796, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 6 ] ], "id": "L3.mlp.1", "node": "l:732", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1011, 1 ] ], "source_columns_b": null, "target_columns": [ [ 846, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 9, 4 ] ], "id": "L3.mlp.2", "node": "l:1039", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1016, 1 ] ], "source_columns_b": null, "target_columns": [ [ 847, 3 ], [ 851, 1 ], [ 1662, 1 ], [ 1797, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 13, 8 ] ], "id": "L3.mlp.3", "node": "l:1015", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1015, 1 ] ], "source_columns_b": null, "target_columns": [ [ 853, 2 ], [ 856, 3 ], [ 1798, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 21, 13 ] ], "id": "L3.mlp.4", "node": "l:991", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 73, 1 ] ], "source_columns_b": null, "target_columns": [ [ 34, 1 ], [ 859, 1 ], [ 862, 3 ], [ 1799, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 34, 17 ] ], "id": "L3.mlp.5", "node": "l:967", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1013, 1 ] ], "source_columns_b": null, "target_columns": [ [ 865, 6 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 51, 18 ] ], "id": "L3.mlp.6", "node": "l:834", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 995, 1 ] ], "source_columns_b": null, "target_columns": [ [ 871, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 69, 18 ] ], "id": "L3.mlp.7", "node": "l:786", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 994, 1 ] ], "source_columns_b": null, "target_columns": [ [ 874, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 87, 9 ] ], "id": "L3.mlp.8", "node": "l:768", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1012, 1 ] ], "source_columns_b": null, "target_columns": [ [ 878, 1 ], [ 885, 2 ], [ 1663, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 96, 2 ] ], "id": "L3.mlp.9", "node": "l:773", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 994, 1 ] ], "source_columns_b": null, "target_columns": [ [ 881, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 98, 2 ] ], "id": "L3.mlp.10", "node": "l:821", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 995, 1 ] ], "source_columns_b": null, "target_columns": [ [ 882, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 100, 5 ] ], "id": "L3.mlp.11", "node": "l:1034", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1016, 1 ] ], "source_columns_b": null, "target_columns": [ [ 883, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 105, 5 ] ], "id": "L3.mlp.12", "node": "l:1010", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1015, 1 ] ], "source_columns_b": null, "target_columns": [ [ 884, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 110, 6 ] ], "id": "L3.mlp.13", "node": "l:986", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 73, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1804, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 116, 6 ] ], "id": "L3.mlp.14", "node": "l:962", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1013, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1635, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 122, 6 ] ], "id": "L3.mlp.15", "node": "l:763", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1012, 1 ] ], "source_columns_b": null, "target_columns": [ [ 887, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 128, 5 ] ], "id": "L3.mlp.16", "node": "l:638", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1010, 1 ] ], "source_columns_b": null, "target_columns": [ [ 72, 1 ], [ 888, 1 ], [ 891, 3 ], [ 896, 14 ], [ 911, 9 ], [ 1805, 3 ], [ 1809, 1 ], [ 1811, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 133, 5 ] ], "id": "L3.mlp.17", "node": "l:490", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1009, 1 ] ], "source_columns_b": null, "target_columns": [ [ 921, 9 ], [ 931, 4 ], [ 936, 4 ], [ 941, 4 ], [ 946, 4 ], [ 951, 3 ], [ 1813, 5 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 138, 26 ] ], "id": "L3.mlp.18", "node": "l:337", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1017, 1 ] ], "source_columns_b": null, "target_columns": [ [ 954, 1 ], [ 956, 4 ], [ 961, 4 ], [ 966, 4 ], [ 971, 9 ], [ 981, 1 ], [ 987, 1 ], [ 999, 4 ], [ 1818, 4 ], [ 1823, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 164, 2 ] ], "id": "L3.mlp.19", "node": "l:16", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1004, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 166, 2 ] ], "id": "L3.mlp.20", "node": "l:29", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1005, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 168, 2 ] ], "id": "L3.mlp.21", "node": "l:80", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1006, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 170, 2 ] ], "id": "L3.mlp.22", "node": "l:81", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1007, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 172, 2 ] ], "id": "L3.mlp.23", "node": "l:215", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1018, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 174, 2 ] ], "id": "L3.mlp.24", "node": "l:216", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1019, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 176, 2 ] ], "id": "L3.mlp.25", "node": "l:265", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1020, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 178, 2 ] ], "id": "L3.mlp.26", "node": "l:266", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1831, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 180, 2 ] ], "id": "L3.mlp.27", "node": "l:297", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1022, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 182, 2 ] ], "id": "L3.mlp.28", "node": "l:298", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1026, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 184, 2 ] ], "id": "L3.mlp.29", "node": "l:331", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1024, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 186, 2 ] ], "id": "L3.mlp.30", "node": "l:332", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1025, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 188, 2 ] ], "id": "L3.mlp.31", "node": "l:371", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1832, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 190, 2 ] ], "id": "L3.mlp.32", "node": "l:372", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1027, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 192, 2 ] ], "id": "L3.mlp.33", "node": "l:408", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1028, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 194, 2 ] ], "id": "L3.mlp.34", "node": "l:409", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1029, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 196, 2 ] ], "id": "L3.mlp.35", "node": "l:448", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1030, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 198, 2 ] ], "id": "L3.mlp.36", "node": "l:449", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1031, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 200, 2 ] ], "id": "L3.mlp.37", "node": "l:484", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1032, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 202, 2 ] ], "id": "L3.mlp.38", "node": "l:485", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1033, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 204, 2 ] ], "id": "L3.mlp.39", "node": "l:518", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1034, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 206, 2 ] ], "id": "L3.mlp.40", "node": "l:519", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1035, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 208, 2 ] ], "id": "L3.mlp.41", "node": "l:551", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1036, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 210, 2 ] ], "id": "L3.mlp.42", "node": "l:552", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1037, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 212, 2 ] ], "id": "L3.mlp.43", "node": "l:590", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1038, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 214, 2 ] ], "id": "L3.mlp.44", "node": "l:591", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1039, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 216, 2 ] ], "id": "L3.mlp.45", "node": "l:626", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1040, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 218, 2 ] ], "id": "L3.mlp.46", "node": "l:627", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1835, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 220, 2 ] ], "id": "L3.mlp.47", "node": "l:646", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1042, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 222, 2 ] ], "id": "L3.mlp.48", "node": "l:647", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1043, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 224, 2 ] ], "id": "L3.mlp.49", "node": "l:706", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1044, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 226, 2 ] ], "id": "L3.mlp.50", "node": "l:707", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1045, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 228, 2 ] ], "id": "L3.mlp.51", "node": "l:755", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1836, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 230, 2 ] ], "id": "L3.mlp.52", "node": "l:756", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1047, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 232, 2 ] ], "id": "L3.mlp.53", "node": "l:803", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1048, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 234, 2 ] ], "id": "L3.mlp.54", "node": "l:804", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1049, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 236, 2 ] ], "id": "L3.mlp.55", "node": "l:851", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1050, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 238, 2 ] ], "id": "L3.mlp.56", "node": "l:852", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1837, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 240, 2 ] ], "id": "L3.mlp.57", "node": "l:877", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1052, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 242, 2 ] ], "id": "L3.mlp.58", "node": "l:878", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1053, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 244, 2 ] ], "id": "L3.mlp.59", "node": "l:894", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1054, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 246, 2 ] ], "id": "L3.mlp.60", "node": "l:895", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1055, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 248, 2 ] ], "id": "L3.mlp.61", "node": "l:924", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1056, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 250, 2 ] ], "id": "L3.mlp.62", "node": "l:925", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1057, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 252, 2 ] ], "id": "L3.mlp.63", "node": "l:948", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1058, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 254, 2 ] ], "id": "L3.mlp.64", "node": "l:949", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1059, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 256, 2 ] ], "id": "L3.mlp.65", "node": "l:972", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1060, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 258, 2 ] ], "id": "L3.mlp.66", "node": "l:973", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 42, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 260, 2 ] ], "id": "L3.mlp.67", "node": "l:996", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1062, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 262, 2 ] ], "id": "L3.mlp.68", "node": "l:997", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1066, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 264, 2 ] ], "id": "L3.mlp.69", "node": "l:1020", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1064, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 266, 2 ] ], "id": "L3.mlp.70", "node": "l:1021", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1065, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 268, 2 ] ], "id": "L3.mlp.71", "node": "l:1044", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1840, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 270, 2 ] ], "id": "L3.mlp.72", "node": "l:1045", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1067, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 272, 2 ] ], "id": "L3.mlp.73", "node": "l:103", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 985, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1068, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 274, 2 ] ], "id": "L3.mlp.74", "node": "l:664", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1069, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 276, 2 ] ], "id": "L3.mlp.75", "node": "l:687", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1070, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 278, 2 ] ], "id": "L3.mlp.76", "node": "l:679", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1636, 3 ], [ 1640, 1 ], [ 1643, 1 ], [ 1664, 17 ], [ 1688, 6 ], [ 1695, 2 ], [ 1702, 2 ], [ 1706, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1071, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 280, 2 ] ], "id": "L3.mlp.77", "node": "l:918", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 836, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1072, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 282, 2 ] ], "id": "L3.mlp.78", "node": "l:658", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 46, 3 ], [ 83, 1 ], [ 1644, 5 ], [ 1694, 1 ], [ 1697, 1 ], [ 1708, 22 ] ], "source_columns_b": null, "target_columns": [ [ 1073, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 284, 2 ] ], "id": "L3.mlp.79", "node": "l:715", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 120, 4 ], [ 1642, 1 ], [ 1649, 4 ], [ 1698, 1 ], [ 1707, 1 ], [ 1730, 22 ] ], "source_columns_b": null, "target_columns": [ [ 1074, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 286, 2 ] ], "id": "L3.mlp.80", "node": "l:812", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 70, 1 ], [ 124, 13 ], [ 138, 1 ], [ 1653, 3 ], [ 1752, 15 ] ], "source_columns_b": null, "target_columns": [ [ 1075, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 288, 2 ] ], "id": "L3.mlp.81", "node": "l:912", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 66, 1 ], [ 71, 1 ], [ 139, 13 ], [ 1656, 3 ], [ 1699, 1 ], [ 1767, 14 ] ], "source_columns_b": null, "target_columns": [ [ 1076, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 290, 2 ] ], "id": "L3.mlp.82", "node": "l:939", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 152, 9 ], [ 828, 8 ], [ 1659, 3 ], [ 1781, 9 ], [ 1791, 4 ] ], "source_columns_b": null, "target_columns": [ [ 1077, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 292, 2 ] ], "id": "L3.mlp.83", "node": "l:695", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1003, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1078, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 294, 2 ] ], "id": "L3.mlp.84", "node": "l:558", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 837, 1 ] ], "source_columns_b": null, "target_columns": [ [ 49, 1 ] ], "type": "add_into_bypass" }, { "hidden_slots": [ [ 296, 2 ] ], "id": "L3.mlp.85", "node": "l:597", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 838, 1 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "add_into_bypass" }, { "hidden_slots": [ [ 298, 2 ] ], "id": "L3.mlp.86", "node": "l:570", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 839, 1 ] ], "source_columns_b": null, "target_columns": [ [ 35, 1 ] ], "type": "add_into_bypass" }, { "hidden_slots": [ [ 300, 2 ] ], "id": "L3.mlp.87", "node": "l:609", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 1795, 1 ] ], "source_columns_b": null, "target_columns": [ [ 36, 1 ] ], "type": "add_into_bypass" }, { "hidden_slots": [ [ 302, 2 ] ], "id": "L3.mlp.88", "node": "l:742", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 841, 1 ] ], "source_columns_b": null, "target_columns": [ [ 52, 1 ] ], "type": "add_into_bypass" }, { "hidden_slots": [ [ 304, 2 ] ], "id": "L3.mlp.89", "node": "l:790", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 842, 1 ] ], "source_columns_b": null, "target_columns": [ [ 53, 1 ] ], "type": "add_into_bypass" }, { "hidden_slots": [ [ 306, 2 ] ], "id": "L3.mlp.90", "node": "l:838", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 843, 1 ] ], "source_columns_b": null, "target_columns": [ [ 54, 1 ] ], "type": "add_into_bypass" }, { "hidden_slots": [ [ 308, 2 ] ], "id": "L3.mlp.91", "node": "l:864", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 844, 1 ] ], "source_columns_b": null, "target_columns": [ [ 37, 1 ] ], "type": "add_into_bypass" }, { "hidden_slots": [], "id": "L3.mlp.92", "node": "l:165", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1079, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.93", "node": "l:99", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1080, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.94", "node": "l:100", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1843, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.95", "node": "l:36", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1082, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.96", "node": "l:62", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1083, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.97", "node": "l:71", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1084, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.98", "node": "l:57", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 44, 1 ], [ 1085, 26 ], [ 1112, 6 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.99", "node": "l:860", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 74, 1 ], [ 1118, 21 ], [ 1140, 11 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.100", "node": "l:900", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1151, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.101", "node": "l:17", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 836, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L3.mlp.102", "node": "l:560", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 837, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L3.mlp.103", "node": "l:599", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 838, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L3.mlp.104", "node": "l:572", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 839, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L3.mlp.105", "node": "l:611", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1795, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L3.mlp.106", "node": "l:744", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 841, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L3.mlp.107", "node": "l:792", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 842, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L3.mlp.108", "node": "l:840", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 843, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L3.mlp.109", "node": "l:866", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 844, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 310, 2 ] ], "id": "L3.mlp.110", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 983, 1 ] ], "source_columns_b": null, "target_columns": [ [ 983, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 312, 2 ] ], "id": "L3.mlp.111", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 984, 1 ] ], "source_columns_b": null, "target_columns": [ [ 984, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:17", "l:39", "l:36", "l:35", "l:38", "l:62", "l:61", "l:64", "l:71", "l:70", "l:73", "l:127", "l:103", "l:165", "l:164", "l:181", "l:99", "l:100", "l:98", "l:101", "l:560", "l:558", "l:599", "l:597", "l:664", "l:659", "l:658", "l:716", "l:715", "l:813", "l:812", "l:57", "l:572", "l:570", "l:611", "l:609", "l:687", "l:679", "l:678", "l:774", "l:773", "l:822", "l:821", "l:860", "l:695", "l:744", "l:742", "l:792", "l:790", "l:840", "l:838", "l:866", "l:864", "l:918", "l:913", "l:912", "l:940", "l:939", "l:900", "l:16", "l:29", "l:15", "l:80", "l:81", "l:79", "l:215", "l:216", "l:214", "l:265", "l:266", "l:264", "l:297", "l:298", "l:296", "l:331", "l:332", "l:330", "l:371", "l:372", "l:370", "l:408", "l:409", "l:407", "l:448", "l:449", "l:447", "l:484", "l:485", "l:483", "l:518", "l:519", "l:517", "l:551", "l:552", "l:550", "l:590", "l:591", "l:589", "l:626", "l:627", "l:625", "l:646", "l:647", "l:645", "l:706", "l:707", "l:705", "l:755", "l:756", "l:754", "l:803", "l:804", "l:802", "l:851", "l:852", "l:850", "l:877", "l:878", "l:876", "l:894", "l:895", "l:893", "l:924", "l:925", "l:923", "l:948", "l:949", "l:947", "l:972", "l:973", "l:971", "l:996", "l:997", "l:995", "l:1020", "l:1021", "l:1019", "l:1044", "l:1045", "l:1043", "l:1039", "l:1034", "l:1015", "l:1010", "l:991", "l:986", "l:967", "l:962", "l:834", "l:786", "l:768", "l:763", "l:732", "l:638", "l:490", "l:337" ] }, { "active_attention_heads": 16, "active_mlp_neurons": 1396, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L4.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:725", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 846, 1 ] ], "source_columns_b": null, "target_columns": [ [ 993, 1 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L4.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:693", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 1078, 1 ] ], "source_columns_b": null, "target_columns": [ [ 51, 1 ] ], "type": "add_into" }, { "heads": [ 2, 1 ], "id": "L4.attn.2", "key_source_columns": [ [ 1796, 1 ], [ 1079, 1 ] ], "node": "l:160", "query_source_columns": [ [ 64, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1796, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1676, 1 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L4.attn.3", "key_source_columns": [ [ 1796, 1 ], [ 69, 1 ] ], "node": "l:123", "query_source_columns": [ [ 67, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1796, 1 ] ], "source_columns_b": null, "target_columns": [ [ 48, 1 ] ], "type": "compute_attn" }, { "heads": [ 4, 1 ], "id": "L4.attn.4", "key_source_columns": [ [ 1796, 1 ], [ 81, 1 ] ], "node": "l:97", "query_source_columns": [ [ 1080, 1 ], [ 1843, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1068, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1677, 1 ] ], "type": "compute_attn" }, { "heads": [ 5, 1 ], "id": "L4.attn.5", "key_source_columns": [ [ 1796, 1 ], [ 162, 1 ] ], "node": "l:34", "query_source_columns": [ [ 1082, 1 ], [ 161, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 38, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1678, 1 ] ], "type": "compute_attn" }, { "heads": [ 6, 1 ], "id": "L4.attn.6", "key_source_columns": [ [ 1796, 1 ], [ 164, 1 ] ], "node": "l:60", "query_source_columns": [ [ 1083, 1 ], [ 163, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 39, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1638, 1 ] ], "type": "compute_attn" }, { "heads": [ 7, 1 ], "id": "L4.attn.7", "key_source_columns": [ [ 1796, 1 ], [ 166, 1 ] ], "node": "l:69", "query_source_columns": [ [ 1084, 1 ], [ 165, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 40, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1679, 1 ] ], "type": "compute_attn" }, { "heads": [ 8, 1 ], "id": "L4.attn.8", "key_source_columns": [ [ 0, 1 ] ], "node": "l:132", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1068, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1680, 1 ] ], "type": "compute_linear" }, { "heads": [ 9, 1 ], "id": "L4.attn.9", "key_source_columns": [ [ 0, 1 ] ], "node": "l:665", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1069, 1 ] ], "source_columns_b": null, "target_columns": [ [ 67, 1 ] ], "type": "compute_linear" }, { "heads": [ 10, 1 ], "id": "L4.attn.10", "key_source_columns": [ [ 0, 1 ] ], "node": "l:688", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1070, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1688, 1 ] ], "type": "compute_linear" }, { "heads": [ 11, 1 ], "id": "L4.attn.11", "key_source_columns": [ [ 0, 1 ] ], "node": "l:919", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1072, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1640, 1 ] ], "type": "compute_linear" }, { "heads": [ 12, 4 ], "id": "L4.attn.12", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 38, 3 ], [ 46, 3 ], [ 64, 8 ], [ 73, 1 ], [ 81, 1 ], [ 83, 1 ], [ 120, 17 ], [ 138, 29 ], [ 828, 12 ], [ 841, 4 ], [ 985, 1 ], [ 994, 4 ], [ 1003, 1 ], [ 1009, 5 ], [ 1015, 3 ], [ 1068, 1 ], [ 1079, 2 ], [ 1082, 3 ], [ 1636, 3 ], [ 1640, 1 ], [ 1642, 20 ], [ 1664, 17 ], [ 1688, 12 ], [ 1702, 2 ], [ 1706, 84 ], [ 1791, 5 ], [ 1843, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 16, "index": 4, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L4.mlp.0", "node": "l:122", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 48, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1689, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L4.mlp.1", "node": "l:171", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 43, 1 ], [ 48, 1 ], [ 986, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1690, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 4 ] ], "id": "L4.mlp.2", "node": "l:180", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1796, 1 ], [ 77, 4 ] ], "source_columns_b": null, "target_columns": [ [ 64, 1 ], [ 1691, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 9, 9 ] ], "id": "L4.mlp.3", "node": "l:199", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1677, 1 ] ], "source_columns_b": null, "target_columns": [ [ 68, 1 ], [ 1643, 1 ], [ 1703, 1 ], [ 1706, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 18, 17 ] ], "id": "L4.mlp.4", "node": "l:96", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1677, 1 ] ], "source_columns_b": null, "target_columns": [ [ 81, 1 ], [ 1709, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 35, 3 ] ], "id": "L4.mlp.5", "node": "l:677", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1071, 1 ], [ 137, 1 ] ], "source_columns_b": null, "target_columns": [ [ 83, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 38, 16 ] ], "id": "L4.mlp.6", "node": "l:662", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1069, 1 ], [ 67, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1645, 1 ], [ 1712, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 54, 12 ] ], "id": "L4.mlp.7", "node": "l:685", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1070, 1 ], [ 1688, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1715, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 66, 24 ] ], "id": "L4.mlp.8", "node": "l:916", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1072, 1 ], [ 1640, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1646, 1 ], [ 1694, 1 ], [ 1718, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 90, 18 ] ], "id": "L4.mlp.9", "node": "l:738", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 993, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1647, 1 ], [ 1722, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 108, 2 ] ], "id": "L4.mlp.10", "node": "l:724", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 993, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1724, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 110, 6 ] ], "id": "L4.mlp.11", "node": "l:938", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1077, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1725, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 116, 14 ] ], "id": "L4.mlp.12", "node": "l:863", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 37, 1 ] ], "source_columns_b": null, "target_columns": [ [ 69, 1 ], [ 120, 1 ], [ 1648, 5 ], [ 1697, 1 ], [ 1707, 1 ], [ 1726, 24 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 130, 14 ] ], "id": "L4.mlp.13", "node": "l:837", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 54, 1 ] ], "source_columns_b": null, "target_columns": [ [ 70, 1 ], [ 121, 14 ], [ 1642, 1 ], [ 1653, 3 ], [ 1698, 1 ], [ 1750, 13 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 144, 14 ] ], "id": "L4.mlp.14", "node": "l:789", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 53, 1 ] ], "source_columns_b": null, "target_columns": [ [ 66, 1 ], [ 135, 2 ], [ 138, 12 ], [ 1656, 2 ], [ 1699, 1 ], [ 1763, 15 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 158, 14 ] ], "id": "L4.mlp.15", "node": "l:741", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 52, 1 ] ], "source_columns_b": null, "target_columns": [ [ 71, 1 ], [ 150, 14 ], [ 831, 1 ], [ 1658, 4 ], [ 1778, 12 ], [ 1791, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 172, 14 ] ], "id": "L4.mlp.16", "node": "l:692", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 51, 1 ] ], "source_columns_b": null, "target_columns": [ [ 38, 2 ], [ 164, 3 ], [ 828, 3 ], [ 832, 2 ], [ 836, 4 ], [ 841, 4 ], [ 983, 3 ], [ 994, 4 ], [ 1664, 4 ], [ 1792, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 186, 14 ] ], "id": "L4.mlp.17", "node": "l:569", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 1 ] ], "source_columns_b": null, "target_columns": [ [ 40, 1 ], [ 46, 1 ], [ 73, 1 ], [ 1003, 1 ], [ 1009, 5 ], [ 1015, 3 ], [ 1068, 1 ], [ 1079, 2 ], [ 1082, 3 ], [ 1636, 2 ], [ 1644, 1 ], [ 1668, 7 ], [ 1695, 2 ], [ 1702, 1 ], [ 1708, 1 ], [ 1843, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 200, 14 ] ], "id": "L4.mlp.18", "node": "l:557", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 49, 1 ] ], "source_columns_b": null, "target_columns": [ [ 47, 1 ], [ 1184, 31 ], [ 1675, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 214, 33 ] ], "id": "L4.mlp.19", "node": "l:32", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 44, 1 ], [ 1085, 26 ], [ 1112, 6 ] ], "source_columns_b": null, "target_columns": [ [ 1215, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 247, 33 ] ], "id": "L4.mlp.20", "node": "l:58", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 234, 33 ] ], "source_columns_b": null, "target_columns": [ [ 75, 1 ], [ 1248, 16 ], [ 1265, 16 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 280, 33 ] ], "id": "L4.mlp.21", "node": "l:67", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 300, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1281, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 313, 33 ] ], "id": "L4.mlp.22", "node": "l:335", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 954, 1 ], [ 956, 4 ], [ 961, 4 ], [ 966, 4 ], [ 971, 9 ], [ 981, 1 ], [ 987, 1 ], [ 999, 4 ], [ 1818, 4 ], [ 1823, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 346, 33 ] ], "id": "L4.mlp.23", "node": "l:488", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 921, 9 ], [ 931, 4 ], [ 936, 4 ], [ 941, 4 ], [ 946, 4 ], [ 951, 3 ], [ 1813, 5 ] ], "source_columns_b": null, "target_columns": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 379, 33 ] ], "id": "L4.mlp.24", "node": "l:630", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 168, 33 ] ], "source_columns_b": null, "target_columns": [ [ 76, 1 ], [ 1380, 2 ], [ 1383, 3 ], [ 1387, 2 ], [ 1390, 17 ], [ 1408, 5 ], [ 1639, 1 ], [ 1683, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 412, 33 ] ], "id": "L4.mlp.25", "node": "l:633", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 267, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1413, 19 ], [ 1433, 13 ], [ 1685, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 445, 33 ] ], "id": "L4.mlp.26", "node": "l:636", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 72, 1 ], [ 888, 1 ], [ 891, 3 ], [ 896, 14 ], [ 911, 9 ], [ 1805, 3 ], [ 1809, 1 ], [ 1811, 1 ] ], "source_columns_b": null, "target_columns": [ [ 58, 1 ], [ 1446, 11 ], [ 1458, 3 ], [ 1462, 17 ], [ 1686, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 478, 33 ] ], "id": "L4.mlp.27", "node": "l:881", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 399, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1479, 3 ], [ 1483, 9 ], [ 1493, 19 ], [ 1687, 1 ], [ 1705, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 511, 33 ] ], "id": "L4.mlp.28", "node": "l:884", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 630, 33 ] ], "source_columns_b": null, "target_columns": [ [ 61, 1 ], [ 1512, 23 ], [ 1536, 9 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 544, 33 ] ], "id": "L4.mlp.29", "node": "l:887", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 333, 33 ] ], "source_columns_b": null, "target_columns": [ [ 62, 1 ], [ 1545, 13 ], [ 1559, 19 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 577, 33 ] ], "id": "L4.mlp.30", "node": "l:898", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 1151, 33 ] ], "source_columns_b": null, "target_columns": [ [ 63, 1 ], [ 1578, 8 ], [ 1587, 24 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 610, 33 ] ], "id": "L4.mlp.31", "node": "l:901", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 663, 33 ] ], "source_columns_b": null, "target_columns": [ [ 65, 1 ], [ 1611, 24 ], [ 1641, 1 ], [ 1700, 1 ], [ 1808, 1 ], [ 1810, 1 ], [ 1812, 1 ], [ 1838, 2 ], [ 1842, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 643, 33 ] ], "id": "L4.mlp.32", "node": "l:931", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 696, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1701, 1 ], [ 1845, 32 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 676, 33 ] ], "id": "L4.mlp.33", "node": "l:952", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 465, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1877, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 709, 33 ] ], "id": "L4.mlp.34", "node": "l:976", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 498, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1910, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 742, 33 ] ], "id": "L4.mlp.35", "node": "l:979", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 729, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1943, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 775, 33 ] ], "id": "L4.mlp.36", "node": "l:1048", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 597, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1976, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 808, 33 ] ], "id": "L4.mlp.37", "node": "l:1051", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 795, 33 ] ], "source_columns_b": null, "target_columns": [ [ 2009, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 841, 33 ] ], "id": "L4.mlp.38", "node": "l:1054", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 366, 33 ] ], "source_columns_b": null, "target_columns": [ [ 834, 2 ], [ 840, 1 ], [ 845, 1 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 889, 2 ], [ 894, 2 ], [ 910, 1 ], [ 920, 1 ], [ 930, 1 ], [ 935, 1 ], [ 940, 1 ], [ 945, 1 ], [ 950, 1 ], [ 955, 1 ], [ 960, 1 ], [ 965, 1 ], [ 970, 1 ], [ 980, 1 ], [ 1014, 1 ], [ 2042, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 874, 2 ] ], "id": "L4.mlp.39", "node": "l:14", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1004, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1021, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 876, 2 ] ], "id": "L4.mlp.40", "node": "l:78", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1006, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1023, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 878, 2 ] ], "id": "L4.mlp.41", "node": "l:213", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1018, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1041, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 880, 2 ] ], "id": "L4.mlp.42", "node": "l:295", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1022, 1 ], [ 1026, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1046, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 882, 2 ] ], "id": "L4.mlp.43", "node": "l:329", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1024, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1051, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 884, 2 ] ], "id": "L4.mlp.44", "node": "l:369", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1832, 1 ], [ 1027, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1061, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 886, 2 ] ], "id": "L4.mlp.45", "node": "l:406", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1028, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1063, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 888, 2 ] ], "id": "L4.mlp.46", "node": "l:446", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1030, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1558, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 890, 2 ] ], "id": "L4.mlp.47", "node": "l:482", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1032, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1432, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 892, 2 ] ], "id": "L4.mlp.48", "node": "l:516", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1034, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1535, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 894, 2 ] ], "id": "L4.mlp.49", "node": "l:549", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1036, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1841, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 896, 2 ] ], "id": "L4.mlp.50", "node": "l:588", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1038, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1457, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 898, 2 ] ], "id": "L4.mlp.51", "node": "l:624", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1040, 1 ], [ 1835, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1586, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 900, 2 ] ], "id": "L4.mlp.52", "node": "l:644", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1042, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1844, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 902, 2 ] ], "id": "L4.mlp.53", "node": "l:704", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1044, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1332, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 904, 2 ] ], "id": "L4.mlp.54", "node": "l:753", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1836, 1 ], [ 1047, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1461, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 906, 2 ] ], "id": "L4.mlp.55", "node": "l:801", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1048, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1081, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 908, 2 ] ], "id": "L4.mlp.56", "node": "l:875", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1052, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1482, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 910, 2 ] ], "id": "L4.mlp.57", "node": "l:892", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1054, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1357, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 912, 2 ] ], "id": "L4.mlp.58", "node": "l:922", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1056, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1492, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 914, 2 ] ], "id": "L4.mlp.59", "node": "l:946", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1058, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1111, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 916, 2 ] ], "id": "L4.mlp.60", "node": "l:970", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1060, 1 ], [ 42, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1369, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 918, 2 ] ], "id": "L4.mlp.61", "node": "l:994", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1062, 1 ], [ 1066, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1382, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 920, 2 ] ], "id": "L4.mlp.62", "node": "l:1018", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1064, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1386, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 922, 2 ] ], "id": "L4.mlp.63", "node": "l:1042", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1840, 1 ], [ 1067, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1389, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 924, 2 ] ], "id": "L4.mlp.64", "node": "l:166", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1796, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1264, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 926, 2 ] ], "id": "L4.mlp.65", "node": "l:133", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1680, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1139, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L4.mlp.66", "node": "l:159", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1407, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L4.mlp.67", "node": "l:132", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1680, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L4.mlp.68", "node": "l:665", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 67, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L4.mlp.69", "node": "l:688", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1688, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L4.mlp.70", "node": "l:919", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1640, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 928, 66 ] ], "id": "L4.mlp.71", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 44, 1 ], [ 1085, 26 ], [ 1112, 6 ] ], "source_columns_b": null, "target_columns": [ [ 44, 1 ], [ 1085, 26 ], [ 1112, 6 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 994, 66 ] ], "id": "L4.mlp.72", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 168, 33 ] ], "source_columns_b": null, "target_columns": [ [ 168, 33 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1060, 66 ] ], "id": "L4.mlp.73", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 234, 33 ] ], "source_columns_b": null, "target_columns": [ [ 234, 33 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1126, 66 ] ], "id": "L4.mlp.74", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 300, 33 ] ], "source_columns_b": null, "target_columns": [ [ 300, 33 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1192, 66 ] ], "id": "L4.mlp.75", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 366, 33 ] ], "source_columns_b": null, "target_columns": [ [ 366, 33 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1258, 66 ] ], "id": "L4.mlp.76", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 921, 9 ], [ 931, 4 ], [ 936, 4 ], [ 941, 4 ], [ 946, 4 ], [ 951, 3 ], [ 1813, 5 ] ], "source_columns_b": null, "target_columns": [ [ 921, 9 ], [ 931, 4 ], [ 936, 4 ], [ 941, 4 ], [ 946, 4 ], [ 951, 3 ], [ 1813, 5 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1324, 8 ] ], "id": "L4.mlp.77", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 77, 4 ] ], "source_columns_b": null, "target_columns": [ [ 77, 4 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1332, 2 ] ], "id": "L4.mlp.78", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1796, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1796, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1334, 2 ] ], "id": "L4.mlp.79", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 49, 1 ] ], "source_columns_b": null, "target_columns": [ [ 49, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1336, 2 ] ], "id": "L4.mlp.80", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 53, 1 ] ], "source_columns_b": null, "target_columns": [ [ 53, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1338, 2 ] ], "id": "L4.mlp.81", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 37, 1 ] ], "source_columns_b": null, "target_columns": [ [ 37, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1340, 2 ] ], "id": "L4.mlp.82", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1072, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1072, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1342, 2 ] ], "id": "L4.mlp.83", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1005, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1005, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1344, 2 ] ], "id": "L4.mlp.84", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1006, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1006, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1346, 2 ] ], "id": "L4.mlp.85", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1007, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1007, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1348, 2 ] ], "id": "L4.mlp.86", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1018, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1018, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1350, 2 ] ], "id": "L4.mlp.87", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1019, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1019, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1352, 2 ] ], "id": "L4.mlp.88", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1025, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1025, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1354, 2 ] ], "id": "L4.mlp.89", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1832, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1832, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1356, 2 ] ], "id": "L4.mlp.90", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1027, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1027, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1358, 2 ] ], "id": "L4.mlp.91", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1029, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1029, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1360, 2 ] ], "id": "L4.mlp.92", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1030, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1030, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1362, 2 ] ], "id": "L4.mlp.93", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1031, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1031, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1364, 2 ] ], "id": "L4.mlp.94", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1032, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1032, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1366, 2 ] ], "id": "L4.mlp.95", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1034, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1034, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1368, 2 ] ], "id": "L4.mlp.96", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1035, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1035, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1370, 2 ] ], "id": "L4.mlp.97", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1036, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1036, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1372, 2 ] ], "id": "L4.mlp.98", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1037, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1037, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1374, 2 ] ], "id": "L4.mlp.99", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1042, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1042, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1376, 2 ] ], "id": "L4.mlp.100", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1043, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1043, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1378, 2 ] ], "id": "L4.mlp.101", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1049, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1049, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1380, 2 ] ], "id": "L4.mlp.102", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1052, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1052, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1382, 2 ] ], "id": "L4.mlp.103", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1053, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1053, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1384, 2 ] ], "id": "L4.mlp.104", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1056, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1056, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1386, 2 ] ], "id": "L4.mlp.105", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1057, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1057, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1388, 2 ] ], "id": "L4.mlp.106", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1060, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1060, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1390, 2 ] ], "id": "L4.mlp.107", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 42, 1 ] ], "source_columns_b": null, "target_columns": [ [ 42, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1392, 2 ] ], "id": "L4.mlp.108", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1062, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1062, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1394, 2 ] ], "id": "L4.mlp.109", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1064, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1064, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:34", "l:60", "l:69", "l:123", "l:122", "l:120", "l:172", "l:171", "l:132", "l:133", "l:131", "l:160", "l:166", "l:159", "l:180", "l:97", "l:665", "l:663", "l:662", "l:688", "l:686", "l:685", "l:677", "l:725", "l:724", "l:693", "l:919", "l:917", "l:916", "l:33", "l:32", "l:59", "l:58", "l:68", "l:67", "l:31", "l:336", "l:335", "l:489", "l:488", "l:556", "l:568", "l:631", "l:630", "l:634", "l:633", "l:637", "l:636", "l:629", "l:691", "l:740", "l:788", "l:836", "l:856", "l:859", "l:862", "l:882", "l:881", "l:885", "l:884", "l:888", "l:887", "l:880", "l:899", "l:898", "l:902", "l:901", "l:929", "l:932", "l:931", "l:953", "l:952", "l:977", "l:976", "l:980", "l:979", "l:1001", "l:1004", "l:1025", "l:1049", "l:1048", "l:1052", "l:1051", "l:1055", "l:1054", "l:1047", "l:14", "l:78", "l:213", "l:295", "l:329", "l:369", "l:406", "l:446", "l:482", "l:516", "l:549", "l:588", "l:624", "l:644", "l:704", "l:753", "l:801", "l:875", "l:892", "l:922", "l:946", "l:970", "l:994", "l:1018", "l:1042", "l:863", "l:837", "l:789", "l:741", "l:692", "l:569", "l:557", "l:938", "l:738", "l:96", "l:199" ] }, { "active_attention_heads": 27, "active_mlp_neurons": 699, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L5.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:676", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 998, 1 ] ], "source_columns_b": null, "target_columns": [ [ 83, 1 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L5.attn.1", "key_source_columns": [ [ 1676, 1 ] ], "node": "l:158", "query_source_columns": [ [ 1407, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1264, 1 ] ], "source_columns_b": null, "target_columns": [ [ 35, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 2 ], "id": "L5.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:30", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1215, 33 ], [ 75, 1 ], [ 1248, 16 ], [ 1265, 49 ] ], "source_columns_b": null, "target_columns": [ [ 37, 1 ], [ 42, 1 ], [ 44, 1 ], [ 48, 2 ], [ 51, 4 ], [ 58, 1 ], [ 61, 2 ], [ 67, 1 ], [ 72, 1 ], [ 75, 6 ], [ 137, 1 ], [ 168, 12 ] ], "type": "compute_linear" }, { "heads": [ 4, 2 ], "id": "L5.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": "l:628", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 76, 1 ], [ 1380, 2 ], [ 1383, 3 ], [ 1387, 2 ], [ 1390, 17 ], [ 1408, 5 ], [ 1639, 1 ], [ 1683, 2 ], [ 1413, 19 ], [ 1433, 13 ], [ 1685, 1 ], [ 58, 1 ], [ 1446, 11 ], [ 1458, 3 ], [ 1462, 17 ], [ 1686, 1 ] ], "source_columns_b": null, "target_columns": [ [ 180, 21 ], [ 234, 12 ] ], "type": "compute_linear" }, { "heads": [ 6, 2 ], "id": "L5.attn.4", "key_source_columns": [ [ 0, 1 ] ], "node": "l:879", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1479, 3 ], [ 1483, 9 ], [ 1493, 19 ], [ 1687, 1 ], [ 1705, 1 ], [ 61, 1 ], [ 1512, 23 ], [ 1536, 9 ], [ 62, 1 ], [ 1545, 13 ], [ 1559, 19 ] ], "source_columns_b": null, "target_columns": [ [ 246, 33 ] ], "type": "compute_linear" }, { "heads": [ 8, 2 ], "id": "L5.attn.5", "key_source_columns": [ [ 0, 1 ] ], "node": "l:1046", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1976, 66 ], [ 834, 2 ], [ 840, 1 ], [ 845, 1 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 889, 2 ], [ 894, 2 ], [ 910, 1 ], [ 920, 1 ], [ 930, 1 ], [ 935, 1 ], [ 940, 1 ], [ 945, 1 ], [ 950, 1 ], [ 955, 1 ], [ 960, 1 ], [ 965, 1 ], [ 970, 1 ], [ 980, 1 ], [ 1014, 1 ], [ 2042, 4 ] ], "source_columns_b": null, "target_columns": [ [ 279, 33 ] ], "type": "compute_linear" }, { "heads": [ 10, 1 ], "id": "L5.attn.6", "key_source_columns": [ [ 0, 1 ] ], "node": "l:661", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1645, 1 ], [ 1712, 3 ] ], "source_columns_b": null, "target_columns": [ [ 312, 4 ] ], "type": "compute_linear" }, { "heads": [ 11, 1 ], "id": "L5.attn.7", "key_source_columns": [ [ 0, 1 ] ], "node": "l:684", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1715, 3 ] ], "source_columns_b": null, "target_columns": [ [ 316, 3 ] ], "type": "compute_linear" }, { "heads": [ 12, 15 ], "id": "L5.attn.8", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 35, 1 ], [ 48, 1 ], [ 51, 2 ], [ 54, 1 ], [ 58, 1 ], [ 61, 2 ], [ 67, 1 ], [ 72, 1 ], [ 75, 2 ], [ 137, 1 ], [ 267, 33 ], [ 333, 33 ], [ 399, 33 ], [ 465, 66 ], [ 597, 165 ], [ 795, 33 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 33 ], [ 930, 1 ], [ 935, 1 ], [ 940, 1 ], [ 945, 1 ], [ 950, 1 ], [ 954, 28 ], [ 986, 2 ], [ 993, 1 ], [ 999, 4 ], [ 1004, 1 ], [ 1014, 1 ], [ 1022, 1 ], [ 1024, 1 ], [ 1026, 1 ], [ 1028, 1 ], [ 1033, 1 ], [ 1038, 3 ], [ 1044, 2 ], [ 1047, 2 ], [ 1054, 2 ], [ 1058, 2 ], [ 1065, 3 ], [ 1069, 3 ], [ 1078, 1 ], [ 1151, 33 ], [ 1215, 99 ], [ 1380, 2 ], [ 1383, 3 ], [ 1387, 2 ], [ 1390, 42 ], [ 1433, 24 ], [ 1458, 3 ], [ 1462, 20 ], [ 1483, 9 ], [ 1493, 42 ], [ 1536, 22 ], [ 1559, 19 ], [ 1639, 2 ], [ 1645, 1 ], [ 1676, 1 ], [ 1683, 6 ], [ 1705, 1 ], [ 1712, 6 ], [ 1805, 3 ], [ 1809, 1 ], [ 1811, 1 ], [ 1818, 4 ], [ 1823, 1 ], [ 1835, 2 ], [ 1840, 1 ], [ 1976, 70 ] ], "type": "cancel" } ], "emitted_attention_heads": 27, "index": 5, "mlp_operations": [ { "hidden_slots": [ [ 1, 32 ] ], "id": "L5.mlp.0", "node": "l:157", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 1 ] ], "source_columns_b": null, "target_columns": [ [ 319, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 33, 12 ] ], "id": "L5.mlp.1", "node": "l:130", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1680, 1 ], [ 1139, 1 ] ], "source_columns_b": null, "target_columns": [ [ 320, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 45, 2 ] ], "id": "L5.mlp.2", "node": "l:119", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 43, 1 ], [ 1689, 1 ] ], "source_columns_b": null, "target_columns": [ [ 323, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 47, 22 ] ], "id": "L5.mlp.3", "node": "l:943", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1077, 1 ] ], "source_columns_b": null, "target_columns": [ [ 324, 6 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 69, 22 ] ], "id": "L5.mlp.4", "node": "l:911", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1076, 1 ] ], "source_columns_b": null, "target_columns": [ [ 330, 6 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 91, 4 ] ], "id": "L5.mlp.5", "node": "l:816", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1075, 1 ] ], "source_columns_b": null, "target_columns": [ [ 336, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 95, 14 ] ], "id": "L5.mlp.6", "node": "l:719", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1074, 1 ] ], "source_columns_b": null, "target_columns": [ [ 340, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 109, 18 ] ], "id": "L5.mlp.7", "node": "l:675", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 83, 1 ] ], "source_columns_b": null, "target_columns": [ [ 344, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 127, 14 ] ], "id": "L5.mlp.8", "node": "l:657", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1073, 1 ] ], "source_columns_b": null, "target_columns": [ [ 347, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 141, 4 ] ], "id": "L5.mlp.9", "node": "l:811", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1075, 1 ] ], "source_columns_b": null, "target_columns": [ [ 351, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 145, 6 ] ], "id": "L5.mlp.10", "node": "l:714", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1074, 1 ] ], "source_columns_b": null, "target_columns": [ [ 352, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 151, 33 ] ], "id": "L5.mlp.11", "node": "l:555", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 47, 1 ], [ 1184, 31 ], [ 1675, 1 ] ], "source_columns_b": null, "target_columns": [ [ 353, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 184, 33 ] ], "id": "L5.mlp.12", "node": "l:567", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 40, 1 ], [ 46, 1 ], [ 73, 1 ], [ 1003, 1 ], [ 1009, 5 ], [ 1015, 3 ], [ 1068, 1 ], [ 1079, 2 ], [ 1082, 3 ], [ 1636, 2 ], [ 1644, 1 ], [ 1668, 7 ], [ 1695, 2 ], [ 1702, 1 ], [ 1708, 1 ], [ 1843, 1 ] ], "source_columns_b": null, "target_columns": [ [ 386, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 217, 33 ] ], "id": "L5.mlp.13", "node": "l:690", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 38, 2 ], [ 164, 3 ], [ 828, 3 ], [ 832, 2 ], [ 836, 4 ], [ 841, 4 ], [ 983, 3 ], [ 994, 4 ], [ 1664, 4 ], [ 1792, 4 ] ], "source_columns_b": null, "target_columns": [ [ 419, 13 ], [ 465, 20 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 250, 33 ] ], "id": "L5.mlp.14", "node": "l:739", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 71, 1 ], [ 150, 14 ], [ 831, 1 ], [ 1658, 4 ], [ 1778, 12 ], [ 1791, 1 ] ], "source_columns_b": null, "target_columns": [ [ 485, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 283, 33 ] ], "id": "L5.mlp.15", "node": "l:787", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 66, 1 ], [ 135, 2 ], [ 138, 12 ], [ 1656, 2 ], [ 1699, 1 ], [ 1763, 15 ] ], "source_columns_b": null, "target_columns": [ [ 518, 13 ], [ 597, 20 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 316, 33 ] ], "id": "L5.mlp.16", "node": "l:855", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 201, 33 ] ], "source_columns_b": null, "target_columns": [ [ 617, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 349, 33 ] ], "id": "L5.mlp.17", "node": "l:858", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 74, 1 ], [ 1118, 21 ], [ 1140, 11 ] ], "source_columns_b": null, "target_columns": [ [ 650, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 382, 33 ] ], "id": "L5.mlp.18", "node": "l:861", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 69, 1 ], [ 120, 1 ], [ 1648, 5 ], [ 1697, 1 ], [ 1707, 1 ], [ 1726, 24 ] ], "source_columns_b": null, "target_columns": [ [ 683, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 415, 2 ] ], "id": "L5.mlp.19", "node": "l:849", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1050, 1 ], [ 1837, 1 ] ], "source_columns_b": null, "target_columns": [ [ 716, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 417, 33 ] ], "id": "L5.mlp.20", "node": "l:12", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1021, 1 ], [ 37, 1 ], [ 42, 1 ], [ 44, 1 ], [ 48, 2 ], [ 51, 4 ], [ 58, 1 ], [ 61, 2 ], [ 67, 1 ], [ 72, 1 ], [ 75, 6 ], [ 137, 1 ], [ 168, 12 ] ], "source_columns_b": null, "target_columns": [ [ 717, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 450, 33 ] ], "id": "L5.mlp.21", "node": "l:622", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1586, 1 ], [ 180, 21 ], [ 234, 12 ] ], "source_columns_b": null, "target_columns": [ [ 750, 12 ], [ 795, 21 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 483, 33 ] ], "id": "L5.mlp.22", "node": "l:873", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1482, 1 ], [ 246, 33 ] ], "source_columns_b": null, "target_columns": [ [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 516, 33 ] ], "id": "L5.mlp.23", "node": "l:1040", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1389, 1 ], [ 279, 33 ] ], "source_columns_b": null, "target_columns": [ [ 896, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 549, 12 ] ], "id": "L5.mlp.24", "node": "l:915", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1646, 1 ], [ 1694, 1 ], [ 1718, 4 ] ], "source_columns_b": null, "target_columns": [ [ 929, 6 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L5.mlp.25", "node": "l:655", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 935, 33 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L5.mlp.26", "node": "l:736", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 968, 14 ], [ 986, 2 ], [ 993, 1 ], [ 999, 4 ], [ 1004, 4 ], [ 1014, 1 ], [ 1018, 2 ], [ 1022, 1 ], [ 1024, 4 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L5.mlp.27", "node": "l:689", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1028, 13 ], [ 1042, 4 ], [ 1047, 3 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 3 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L5.mlp.28", "node": "l:909", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1067, 1 ], [ 1069, 4 ], [ 1078, 1 ], [ 1085, 26 ], [ 1112, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L5.mlp.29", "node": "l:654", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1113, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L5.mlp.30", "node": "l:661", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 312, 4 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L5.mlp.31", "node": "l:684", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 316, 3 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 561, 66 ] ], "id": "L5.mlp.32", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 66, 1 ], [ 135, 2 ], [ 138, 12 ], [ 1656, 2 ], [ 1699, 1 ], [ 1763, 15 ] ], "source_columns_b": null, "target_columns": [ [ 66, 1 ], [ 135, 2 ], [ 138, 12 ], [ 1656, 2 ], [ 1699, 1 ], [ 1763, 15 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 627, 66 ] ], "id": "L5.mlp.33", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 38, 2 ], [ 164, 3 ], [ 828, 3 ], [ 832, 2 ], [ 836, 4 ], [ 841, 4 ], [ 983, 3 ], [ 994, 4 ], [ 1664, 4 ], [ 1792, 4 ] ], "source_columns_b": null, "target_columns": [ [ 38, 2 ], [ 164, 3 ], [ 828, 3 ], [ 832, 2 ], [ 836, 4 ], [ 841, 4 ], [ 983, 3 ], [ 994, 4 ], [ 1664, 4 ], [ 1792, 4 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 693, 2 ] ], "id": "L5.mlp.34", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 43, 1 ] ], "source_columns_b": null, "target_columns": [ [ 43, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 695, 2 ] ], "id": "L5.mlp.35", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1689, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1689, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 697, 2 ] ], "id": "L5.mlp.36", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1482, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1482, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:119", "l:130", "l:139", "l:158", "l:157", "l:661", "l:655", "l:684", "l:736", "l:689", "l:676", "l:915", "l:909", "l:30", "l:555", "l:567", "l:628", "l:690", "l:739", "l:787", "l:855", "l:858", "l:861", "l:854", "l:879", "l:1046", "l:13", "l:12", "l:623", "l:622", "l:849", "l:874", "l:873", "l:1041", "l:1040", "l:943", "l:911", "l:816", "l:811", "l:719", "l:714", "l:675", "l:657", "l:654" ] }, { "active_attention_heads": 27, "active_mlp_neurons": 1243, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L6.attn.0", "key_source_columns": [ [ 929, 6 ] ], "node": "l:910", "query_source_columns": [ [ 330, 6 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 34, 2 ], [ 37, 4 ], [ 42, 3 ], [ 46, 4 ], [ 51, 4 ], [ 58, 1 ], [ 61, 2 ], [ 66, 2 ], [ 69, 1 ], [ 71, 10 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L6.attn.1", "key_source_columns": [ [ 929, 6 ] ], "node": "l:942", "query_source_columns": [ [ 324, 6 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 83, 1 ], [ 120, 1 ], [ 135, 31 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L6.attn.2", "key_source_columns": [ [ 929, 6 ] ], "node": "l:966", "query_source_columns": [ [ 865, 6 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 166, 1 ], [ 168, 32 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L6.attn.3", "key_source_columns": [ [ 929, 6 ] ], "node": "l:990", "query_source_columns": [ [ 34, 1 ], [ 859, 1 ], [ 862, 3 ], [ 1799, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 200, 33 ] ], "type": "compute_attn" }, { "heads": [ 4, 1 ], "id": "L6.attn.4", "key_source_columns": [ [ 929, 6 ] ], "node": "l:1014", "query_source_columns": [ [ 853, 2 ], [ 856, 3 ], [ 1798, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 233, 33 ] ], "type": "compute_attn" }, { "heads": [ 5, 1 ], "id": "L6.attn.5", "key_source_columns": [ [ 929, 6 ] ], "node": "l:1038", "query_source_columns": [ [ 847, 3 ], [ 851, 1 ], [ 1662, 1 ], [ 1797, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 266, 33 ] ], "type": "compute_attn" }, { "heads": [ 6, 1 ], "id": "L6.attn.6", "key_source_columns": [ [ 312, 4 ] ], "node": "l:656", "query_source_columns": [ [ 347, 4 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 299, 24 ], [ 324, 9 ] ], "type": "compute_attn" }, { "heads": [ 7, 1 ], "id": "L6.attn.7", "key_source_columns": [ [ 312, 4 ] ], "node": "l:718", "query_source_columns": [ [ 340, 4 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 333, 18 ], [ 617, 15 ] ], "type": "compute_attn" }, { "heads": [ 8, 1 ], "id": "L6.attn.8", "key_source_columns": [ [ 312, 4 ] ], "node": "l:767", "query_source_columns": [ [ 878, 1 ], [ 885, 2 ], [ 1663, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 632, 33 ] ], "type": "compute_attn" }, { "heads": [ 9, 1 ], "id": "L6.attn.9", "key_source_columns": [ [ 312, 4 ] ], "node": "l:815", "query_source_columns": [ [ 336, 4 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 665, 33 ] ], "type": "compute_attn" }, { "heads": [ 10, 1 ], "id": "L6.attn.10", "key_source_columns": [ [ 316, 3 ] ], "node": "l:674", "query_source_columns": [ [ 344, 3 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 698, 18 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 1 ] ], "type": "compute_attn" }, { "heads": [ 11, 1 ], "id": "L6.attn.11", "key_source_columns": [ [ 316, 3 ] ], "node": "l:737", "query_source_columns": [ [ 1647, 1 ], [ 1722, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 848, 2 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 15 ], [ 878, 1 ], [ 885, 2 ], [ 929, 6 ] ], "type": "compute_attn" }, { "heads": [ 12, 1 ], "id": "L6.attn.12", "key_source_columns": [ [ 316, 3 ] ], "node": "l:785", "query_source_columns": [ [ 874, 3 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 983, 3 ], [ 994, 5 ], [ 1003, 1 ], [ 1009, 5 ], [ 1015, 3 ], [ 1021, 1 ], [ 1050, 1 ], [ 1068, 1 ], [ 1073, 5 ], [ 1079, 2 ], [ 1082, 3 ], [ 1114, 3 ] ], "type": "compute_attn" }, { "heads": [ 13, 1 ], "id": "L6.attn.13", "key_source_columns": [ [ 316, 3 ] ], "node": "l:833", "query_source_columns": [ [ 871, 3 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1117, 33 ] ], "type": "compute_attn" }, { "heads": [ 14, 2 ], "id": "L6.attn.14", "key_source_columns": [ [ 0, 1 ] ], "node": "l:853", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 617, 99 ] ], "source_columns_b": null, "target_columns": [ [ 1150, 33 ] ], "type": "compute_linear" }, { "heads": [ 16, 1 ], "id": "L6.attn.15", "key_source_columns": [ [ 0, 1 ] ], "node": "l:156", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 319, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1183, 1 ] ], "type": "compute_linear" }, { "heads": [ 17, 1 ], "id": "L6.attn.16", "key_source_columns": [ [ 0, 1 ] ], "node": "l:129", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 320, 3 ] ], "source_columns_b": null, "target_columns": [ [ 1184, 3 ] ], "type": "compute_linear" }, { "heads": [ 18, 9 ], "id": "L6.attn.17", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 2 ], [ 37, 1 ], [ 40, 1 ], [ 42, 1 ], [ 44, 1 ], [ 46, 4 ], [ 51, 4 ], [ 58, 1 ], [ 61, 2 ], [ 67, 1 ], [ 69, 1 ], [ 71, 10 ], [ 83, 1 ], [ 120, 1 ], [ 137, 1 ], [ 150, 14 ], [ 168, 155 ], [ 324, 27 ], [ 617, 99 ], [ 831, 1 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 15 ], [ 878, 1 ], [ 885, 2 ], [ 929, 6 ], [ 998, 1 ], [ 1003, 1 ], [ 1009, 5 ], [ 1015, 3 ], [ 1021, 1 ], [ 1050, 1 ], [ 1068, 1 ], [ 1073, 5 ], [ 1079, 2 ], [ 1082, 3 ], [ 1118, 33 ], [ 1184, 31 ], [ 1389, 1 ], [ 1586, 1 ], [ 1636, 2 ], [ 1644, 1 ], [ 1646, 7 ], [ 1658, 6 ], [ 1668, 8 ], [ 1680, 1 ], [ 1694, 4 ], [ 1702, 1 ], [ 1707, 2 ], [ 1718, 6 ], [ 1726, 24 ], [ 1778, 12 ], [ 1791, 1 ], [ 1797, 3 ], [ 1837, 1 ], [ 1843, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 27, "index": 6, "mlp_operations": [ { "hidden_slots": [ [ 1, 3 ] ], "id": "L6.mlp.0", "node": "l:117", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 323, 1 ], [ 1184, 3 ] ], "source_columns_b": null, "target_columns": [ [ 1187, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 4, 66 ] ], "id": "L6.mlp.1", "node": "l:138", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 323, 1 ], [ 1, 33 ], [ 85, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1190, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 70, 66 ] ], "id": "L6.mlp.2", "node": "l:652", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1113, 1 ], [ 935, 33 ], [ 299, 24 ], [ 324, 9 ] ], "source_columns_b": null, "target_columns": [ [ 1223, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 136, 66 ] ], "id": "L6.mlp.3", "node": "l:712", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 352, 1 ], [ 935, 33 ], [ 333, 18 ], [ 617, 15 ] ], "source_columns_b": null, "target_columns": [ [ 1256, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 202, 66 ] ], "id": "L6.mlp.4", "node": "l:761", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 887, 1 ], [ 935, 33 ], [ 632, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1289, 25 ], [ 1380, 2 ], [ 1383, 3 ], [ 1387, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 268, 66 ] ], "id": "L6.mlp.5", "node": "l:809", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 351, 1 ], [ 935, 33 ], [ 665, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1390, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 334, 66 ] ], "id": "L6.mlp.6", "node": "l:668", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1008, 1 ], [ 698, 18 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 1 ], [ 1028, 13 ], [ 1042, 4 ], [ 1047, 3 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 3 ] ], "source_columns_b": null, "target_columns": [ [ 1423, 9 ], [ 1433, 24 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 400, 66 ] ], "id": "L6.mlp.7", "node": "l:722", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1724, 1 ], [ 968, 14 ], [ 986, 2 ], [ 993, 1 ], [ 999, 4 ], [ 1004, 4 ], [ 1014, 1 ], [ 1018, 2 ], [ 1022, 1 ], [ 1024, 4 ], [ 848, 2 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 15 ], [ 878, 1 ], [ 885, 2 ], [ 929, 6 ] ], "source_columns_b": null, "target_columns": [ [ 1458, 3 ], [ 1462, 30 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 466, 66 ] ], "id": "L6.mlp.8", "node": "l:771", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 881, 1 ], [ 968, 14 ], [ 986, 2 ], [ 993, 1 ], [ 999, 4 ], [ 1004, 4 ], [ 1014, 1 ], [ 1018, 2 ], [ 1022, 1 ], [ 1024, 4 ], [ 983, 3 ], [ 994, 5 ], [ 1003, 1 ], [ 1009, 5 ], [ 1015, 3 ], [ 1021, 1 ], [ 1050, 1 ], [ 1068, 1 ], [ 1073, 5 ], [ 1079, 2 ], [ 1082, 3 ], [ 1114, 3 ] ], "source_columns_b": null, "target_columns": [ [ 1493, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 532, 66 ] ], "id": "L6.mlp.9", "node": "l:819", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 882, 1 ], [ 968, 14 ], [ 986, 2 ], [ 993, 1 ], [ 999, 4 ], [ 1004, 4 ], [ 1014, 1 ], [ 1018, 2 ], [ 1022, 1 ], [ 1024, 4 ], [ 1117, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1526, 9 ], [ 1536, 22 ], [ 1559, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 598, 66 ] ], "id": "L6.mlp.10", "node": "l:906", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 167, 1 ], [ 1067, 1 ], [ 1069, 4 ], [ 1078, 1 ], [ 1085, 26 ], [ 1112, 1 ], [ 34, 2 ], [ 37, 4 ], [ 42, 3 ], [ 46, 4 ], [ 51, 4 ], [ 58, 1 ], [ 61, 2 ], [ 66, 2 ], [ 69, 1 ], [ 71, 10 ] ], "source_columns_b": null, "target_columns": [ [ 1561, 17 ], [ 1586, 1 ], [ 1636, 2 ], [ 1639, 2 ], [ 1644, 9 ], [ 1656, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 664, 66 ] ], "id": "L6.mlp.11", "node": "l:936", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1725, 1 ], [ 1067, 1 ], [ 1069, 4 ], [ 1078, 1 ], [ 1085, 26 ], [ 1112, 1 ], [ 83, 1 ], [ 120, 1 ], [ 135, 31 ] ], "source_columns_b": null, "target_columns": [ [ 1658, 19 ], [ 1680, 1 ], [ 1683, 7 ], [ 1694, 4 ], [ 1699, 1 ], [ 1702, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 730, 66 ] ], "id": "L6.mlp.12", "node": "l:960", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1635, 1 ], [ 1067, 1 ], [ 1069, 4 ], [ 1078, 1 ], [ 1085, 26 ], [ 1112, 1 ], [ 166, 1 ], [ 168, 32 ] ], "source_columns_b": null, "target_columns": [ [ 1705, 1 ], [ 1707, 2 ], [ 1712, 12 ], [ 1726, 18 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 796, 66 ] ], "id": "L6.mlp.13", "node": "l:984", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1804, 1 ], [ 1067, 1 ], [ 1069, 4 ], [ 1078, 1 ], [ 1085, 26 ], [ 1112, 1 ], [ 200, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1744, 6 ], [ 1763, 27 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 862, 66 ] ], "id": "L6.mlp.14", "node": "l:1008", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 884, 1 ], [ 1067, 1 ], [ 1069, 4 ], [ 1078, 1 ], [ 1085, 26 ], [ 1112, 1 ], [ 233, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1791, 9 ], [ 1805, 3 ], [ 1809, 1 ], [ 1811, 1 ], [ 1813, 9 ], [ 1823, 1 ], [ 1832, 1 ], [ 1835, 3 ], [ 1840, 1 ], [ 1843, 1 ], [ 1976, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 928, 66 ] ], "id": "L6.mlp.15", "node": "l:1032", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 883, 1 ], [ 1067, 1 ], [ 1069, 4 ], [ 1078, 1 ], [ 1085, 26 ], [ 1112, 1 ], [ 266, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1979, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 994, 33 ] ], "id": "L6.mlp.16", "node": "l:847", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 716, 1 ], [ 1150, 33 ] ], "source_columns_b": null, "target_columns": [ [ 2012, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1027, 2 ] ], "id": "L6.mlp.17", "node": "l:177", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1183, 1 ] ], "source_columns_b": null, "target_columns": [ [ 2045, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L6.mlp.18", "node": "l:156", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1183, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L6.mlp.19", "node": "l:129", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1184, 3 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 1029, 66 ] ], "id": "L6.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 935, 33 ] ], "source_columns_b": null, "target_columns": [ [ 935, 33 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1095, 66 ] ], "id": "L6.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1028, 13 ], [ 1042, 4 ], [ 1047, 3 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 3 ] ], "source_columns_b": null, "target_columns": [ [ 1028, 13 ], [ 1042, 4 ], [ 1047, 3 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1161, 66 ] ], "id": "L6.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1067, 1 ], [ 1069, 4 ], [ 1078, 1 ], [ 1085, 26 ], [ 1112, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1067, 1 ], [ 1069, 4 ], [ 1078, 1 ], [ 1085, 26 ], [ 1112, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1227, 2 ] ], "id": "L6.mlp.23", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 323, 1 ] ], "source_columns_b": null, "target_columns": [ [ 323, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1229, 2 ] ], "id": "L6.mlp.24", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1008, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1008, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1231, 2 ] ], "id": "L6.mlp.25", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 881, 1 ] ], "source_columns_b": null, "target_columns": [ [ 881, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1233, 2 ] ], "id": "L6.mlp.26", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 884, 1 ] ], "source_columns_b": null, "target_columns": [ [ 884, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1235, 2 ] ], "id": "L6.mlp.27", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1725, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1725, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1237, 2 ] ], "id": "L6.mlp.28", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 167, 1 ] ], "source_columns_b": null, "target_columns": [ [ 167, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1239, 2 ] ], "id": "L6.mlp.29", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 351, 1 ] ], "source_columns_b": null, "target_columns": [ [ 351, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1241, 2 ] ], "id": "L6.mlp.30", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 352, 1 ] ], "source_columns_b": null, "target_columns": [ [ 352, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:129", "l:118", "l:117", "l:116", "l:138", "l:156", "l:177", "l:656", "l:653", "l:652", "l:718", "l:713", "l:712", "l:767", "l:762", "l:761", "l:815", "l:810", "l:809", "l:674", "l:669", "l:668", "l:737", "l:723", "l:722", "l:785", "l:772", "l:771", "l:833", "l:820", "l:819", "l:910", "l:907", "l:906", "l:942", "l:937", "l:936", "l:966", "l:961", "l:960", "l:990", "l:985", "l:984", "l:1014", "l:1009", "l:1008", "l:1038", "l:1033", "l:1032", "l:651", "l:667", "l:711", "l:721", "l:760", "l:770", "l:808", "l:818", "l:853", "l:905", "l:935", "l:959", "l:983", "l:1007", "l:1031", "l:848", "l:847" ] }, { "active_attention_heads": 11, "active_mlp_neurons": 1527, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L7.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:115", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 1, "role": "transport", "source_columns": [ [ 1187, 3 ], [ 84, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1800, 4 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L7.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:178", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 2045, 1 ] ], "source_columns_b": null, "target_columns": [ [ 34, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 9 ], "id": "L7.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 2 ], [ 37, 4 ], [ 42, 3 ], [ 46, 4 ], [ 51, 4 ], [ 58, 1 ], [ 61, 2 ], [ 66, 2 ], [ 69, 1 ], [ 71, 10 ], [ 83, 1 ], [ 85, 33 ], [ 120, 1 ], [ 135, 32 ], [ 168, 155 ], [ 324, 27 ], [ 617, 100 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 15 ], [ 878, 1 ], [ 882, 2 ], [ 885, 3 ], [ 929, 6 ], [ 968, 14 ], [ 983, 5 ], [ 993, 15 ], [ 1009, 11 ], [ 1021, 2 ], [ 1024, 4 ], [ 1050, 1 ], [ 1068, 1 ], [ 1073, 5 ], [ 1079, 2 ], [ 1082, 3 ], [ 1113, 70 ], [ 1184, 3 ], [ 1635, 1 ], [ 1724, 1 ], [ 1804, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 11, "index": 7, "mlp_operations": [ { "hidden_slots": [ [ 1, 12 ] ], "id": "L7.mlp.0", "node": "l:175", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 2045, 1 ], [ 34, 1 ] ], "source_columns_b": null, "target_columns": [ [ 35, 1 ], [ 37, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 13, 13 ] ], "id": "L7.mlp.1", "node": "l:232", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1677, 1 ] ], "source_columns_b": null, "target_columns": [ [ 39, 2 ], [ 42, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 26, 14 ] ], "id": "L7.mlp.2", "node": "l:596", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 50, 1 ] ], "source_columns_b": null, "target_columns": [ [ 44, 1 ], [ 46, 4 ], [ 51, 4 ], [ 58, 1 ], [ 61, 2 ], [ 66, 2 ], [ 69, 1 ], [ 71, 10 ], [ 83, 1 ], [ 85, 7 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 40, 33 ] ], "id": "L7.mlp.3", "node": "l:650", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 1223, 33 ] ], "source_columns_b": null, "target_columns": [ [ 92, 26 ], [ 120, 1 ], [ 135, 6 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 73, 33 ] ], "id": "L7.mlp.4", "node": "l:666", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 1423, 9 ], [ 1433, 24 ] ], "source_columns_b": null, "target_columns": [ [ 141, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 106, 33 ] ], "id": "L7.mlp.5", "node": "l:710", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 1256, 33 ] ], "source_columns_b": null, "target_columns": [ [ 174, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 139, 33 ] ], "id": "L7.mlp.6", "node": "l:720", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 1458, 3 ], [ 1462, 30 ] ], "source_columns_b": null, "target_columns": [ [ 207, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 172, 33 ] ], "id": "L7.mlp.7", "node": "l:759", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 1289, 25 ], [ 1380, 2 ], [ 1383, 3 ], [ 1387, 3 ] ], "source_columns_b": null, "target_columns": [ [ 240, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 205, 33 ] ], "id": "L7.mlp.8", "node": "l:769", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 1493, 33 ] ], "source_columns_b": null, "target_columns": [ [ 273, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 238, 33 ] ], "id": "L7.mlp.9", "node": "l:807", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 1390, 33 ] ], "source_columns_b": null, "target_columns": [ [ 306, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 271, 33 ] ], "id": "L7.mlp.10", "node": "l:817", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 1526, 9 ], [ 1536, 22 ], [ 1559, 2 ] ], "source_columns_b": null, "target_columns": [ [ 339, 14 ], [ 617, 19 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 304, 33 ] ], "id": "L7.mlp.11", "node": "l:835", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 70, 1 ], [ 121, 14 ], [ 1642, 1 ], [ 1653, 3 ], [ 1698, 1 ], [ 1750, 13 ] ], "source_columns_b": null, "target_columns": [ [ 636, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 337, 33 ] ], "id": "L7.mlp.12", "node": "l:904", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 1561, 17 ], [ 1586, 1 ], [ 1636, 2 ], [ 1639, 2 ], [ 1644, 9 ], [ 1656, 2 ] ], "source_columns_b": null, "target_columns": [ [ 669, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 370, 33 ] ], "id": "L7.mlp.13", "node": "l:928", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 432, 33 ] ], "source_columns_b": null, "target_columns": [ [ 702, 15 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 403, 33 ] ], "id": "L7.mlp.14", "node": "l:934", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 1658, 19 ], [ 1680, 1 ], [ 1683, 7 ], [ 1694, 4 ], [ 1699, 1 ], [ 1702, 1 ] ], "source_columns_b": null, "target_columns": [ [ 853, 2 ], [ 856, 4 ], [ 862, 15 ], [ 878, 1 ], [ 881, 7 ], [ 929, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 436, 33 ] ], "id": "L7.mlp.15", "node": "l:958", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 1705, 1 ], [ 1707, 2 ], [ 1712, 12 ], [ 1726, 18 ] ], "source_columns_b": null, "target_columns": [ [ 933, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 469, 33 ] ], "id": "L7.mlp.16", "node": "l:982", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 1744, 6 ], [ 1763, 27 ] ], "source_columns_b": null, "target_columns": [ [ 966, 16 ], [ 983, 5 ], [ 993, 12 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 502, 33 ] ], "id": "L7.mlp.17", "node": "l:1000", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 531, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1005, 15 ], [ 1021, 2 ], [ 1024, 16 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 535, 33 ] ], "id": "L7.mlp.18", "node": "l:1003", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 762, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1040, 1 ], [ 1042, 4 ], [ 1047, 4 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 14 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 568, 33 ] ], "id": "L7.mlp.19", "node": "l:1006", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 1791, 9 ], [ 1805, 3 ], [ 1809, 1 ], [ 1811, 1 ], [ 1813, 9 ], [ 1823, 1 ], [ 1832, 1 ], [ 1835, 3 ], [ 1840, 1 ], [ 1843, 1 ], [ 1976, 3 ] ], "source_columns_b": null, "target_columns": [ [ 1078, 3 ], [ 1082, 29 ], [ 1112, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 601, 33 ] ], "id": "L7.mlp.20", "node": "l:1024", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 564, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1113, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 634, 33 ] ], "id": "L7.mlp.21", "node": "l:1030", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 1979, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1146, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L7.mlp.22", "node": "l:152", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1179, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L7.mlp.23", "node": "l:153", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1180, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L7.mlp.24", "node": "l:228", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1181, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L7.mlp.25", "node": "l:230", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1182, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L7.mlp.26", "node": "l:202", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1184, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L7.mlp.27", "node": "l:203", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1185, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L7.mlp.28", "node": "l:205", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1186, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L7.mlp.29", "node": "l:236", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1635, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L7.mlp.30", "node": "l:237", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1724, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L7.mlp.31", "node": "l:239", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1725, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L7.mlp.32", "node": "l:92", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1804, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L7.mlp.33", "node": "l:178", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 667, 66 ] ], "id": "L7.mlp.34", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1223, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1223, 33 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 733, 66 ] ], "id": "L7.mlp.35", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1256, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1256, 33 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 799, 66 ] ], "id": "L7.mlp.36", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1289, 25 ], [ 1380, 2 ], [ 1383, 3 ], [ 1387, 3 ] ], "source_columns_b": null, "target_columns": [ [ 1289, 25 ], [ 1380, 2 ], [ 1383, 3 ], [ 1387, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 865, 66 ] ], "id": "L7.mlp.37", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1390, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1390, 33 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 931, 66 ] ], "id": "L7.mlp.38", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1423, 9 ], [ 1433, 24 ] ], "source_columns_b": null, "target_columns": [ [ 1423, 9 ], [ 1433, 24 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 997, 66 ] ], "id": "L7.mlp.39", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1458, 3 ], [ 1462, 30 ] ], "source_columns_b": null, "target_columns": [ [ 1458, 3 ], [ 1462, 30 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1063, 66 ] ], "id": "L7.mlp.40", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1493, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1493, 33 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1129, 66 ] ], "id": "L7.mlp.41", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1526, 9 ], [ 1536, 22 ], [ 1559, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1526, 9 ], [ 1536, 22 ], [ 1559, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1195, 66 ] ], "id": "L7.mlp.42", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1658, 19 ], [ 1680, 1 ], [ 1683, 7 ], [ 1694, 4 ], [ 1699, 1 ], [ 1702, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1658, 19 ], [ 1680, 1 ], [ 1683, 7 ], [ 1694, 4 ], [ 1699, 1 ], [ 1702, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1261, 66 ] ], "id": "L7.mlp.43", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1705, 1 ], [ 1707, 2 ], [ 1712, 12 ], [ 1726, 18 ] ], "source_columns_b": null, "target_columns": [ [ 1705, 1 ], [ 1707, 2 ], [ 1712, 12 ], [ 1726, 18 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1327, 66 ] ], "id": "L7.mlp.44", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1744, 6 ], [ 1763, 27 ] ], "source_columns_b": null, "target_columns": [ [ 1744, 6 ], [ 1763, 27 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1393, 66 ] ], "id": "L7.mlp.45", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1791, 9 ], [ 1805, 3 ], [ 1809, 1 ], [ 1811, 1 ], [ 1813, 9 ], [ 1823, 1 ], [ 1832, 1 ], [ 1835, 3 ], [ 1840, 1 ], [ 1843, 1 ], [ 1976, 3 ] ], "source_columns_b": null, "target_columns": [ [ 1791, 9 ], [ 1805, 3 ], [ 1809, 1 ], [ 1811, 1 ], [ 1813, 9 ], [ 1823, 1 ], [ 1832, 1 ], [ 1835, 3 ], [ 1840, 1 ], [ 1843, 1 ], [ 1976, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1459, 66 ] ], "id": "L7.mlp.46", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1979, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1979, 33 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1525, 2 ] ], "id": "L7.mlp.47", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 50, 1 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:115", "l:178", "l:176", "l:175", "l:152", "l:153", "l:151", "l:228", "l:230", "l:229", "l:92", "l:202", "l:203", "l:205", "l:201", "l:204", "l:236", "l:237", "l:239", "l:235", "l:238", "l:595", "l:650", "l:666", "l:649", "l:710", "l:720", "l:709", "l:759", "l:769", "l:758", "l:807", "l:817", "l:835", "l:806", "l:904", "l:897", "l:928", "l:934", "l:927", "l:958", "l:982", "l:975", "l:1000", "l:1003", "l:1006", "l:999", "l:1024", "l:1030", "l:596", "l:232" ] }, { "active_attention_heads": 32, "active_mlp_neurons": 459, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L8.attn.0", "key_source_columns": [ [ 1800, 4 ] ], "node": "l:198", "query_source_columns": [ [ 68, 1 ], [ 1643, 1 ], [ 1703, 1 ], [ 1706, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1190, 33 ] ], "source_columns_b": null, "target_columns": [ [ 34, 2 ], [ 37, 4 ], [ 42, 2 ], [ 50, 1 ], [ 68, 1 ], [ 70, 1 ], [ 81, 1 ], [ 84, 1 ], [ 92, 20 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L8.attn.1", "key_source_columns": [ [ 1800, 4 ] ], "node": "l:231", "query_source_columns": [ [ 39, 2 ], [ 42, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1190, 33 ] ], "source_columns_b": null, "target_columns": [ [ 112, 6 ], [ 120, 27 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L8.attn.2", "key_source_columns": [ [ 1800, 4 ] ], "node": "l:95", "query_source_columns": [ [ 81, 1 ], [ 1709, 3 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1190, 33 ] ], "source_columns_b": null, "target_columns": [ [ 147, 33 ] ], "type": "compute_attn" }, { "heads": [ 3, 2 ], "id": "L8.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": "l:648", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 92, 26 ], [ 120, 1 ], [ 135, 39 ], [ 419, 13 ], [ 465, 20 ] ], "source_columns_b": null, "target_columns": [ [ 180, 33 ] ], "type": "compute_linear" }, { "heads": [ 5, 2 ], "id": "L8.attn.4", "key_source_columns": [ [ 0, 1 ] ], "node": "l:708", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 174, 66 ], [ 485, 33 ] ], "source_columns_b": null, "target_columns": [ [ 213, 33 ] ], "type": "compute_linear" }, { "heads": [ 7, 2 ], "id": "L8.attn.5", "key_source_columns": [ [ 0, 1 ] ], "node": "l:757", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 240, 66 ], [ 518, 13 ], [ 597, 20 ] ], "source_columns_b": null, "target_columns": [ [ 246, 33 ] ], "type": "compute_linear" }, { "heads": [ 9, 2 ], "id": "L8.attn.6", "key_source_columns": [ [ 0, 1 ] ], "node": "l:805", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 306, 47 ], [ 617, 52 ] ], "source_columns_b": null, "target_columns": [ [ 279, 33 ] ], "type": "compute_linear" }, { "heads": [ 11, 2 ], "id": "L8.attn.7", "key_source_columns": [ [ 0, 1 ] ], "node": "l:926", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 702, 15 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 1701, 1 ], [ 1845, 32 ], [ 853, 2 ], [ 856, 4 ], [ 862, 15 ], [ 878, 1 ], [ 881, 7 ], [ 929, 4 ] ], "source_columns_b": null, "target_columns": [ [ 312, 33 ] ], "type": "compute_linear" }, { "heads": [ 13, 2 ], "id": "L8.attn.8", "key_source_columns": [ [ 0, 1 ] ], "node": "l:974", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1910, 66 ], [ 966, 16 ], [ 983, 5 ], [ 993, 12 ] ], "source_columns_b": null, "target_columns": [ [ 345, 8 ], [ 419, 25 ] ], "type": "compute_linear" }, { "heads": [ 15, 2 ], "id": "L8.attn.9", "key_source_columns": [ [ 0, 1 ] ], "node": "l:998", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1005, 15 ], [ 1021, 2 ], [ 1024, 17 ], [ 1042, 4 ], [ 1047, 4 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 17 ], [ 1082, 29 ], [ 1112, 1 ] ], "source_columns_b": null, "target_columns": [ [ 444, 33 ] ], "type": "compute_linear" }, { "heads": [ 17, 1 ], "id": "L8.attn.10", "key_source_columns": [ [ 0, 1 ] ], "node": "l:174", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 35, 1 ], [ 37, 2 ] ], "source_columns_b": null, "target_columns": [ [ 477, 3 ] ], "type": "compute_linear" }, { "heads": [ 18, 14 ], "id": "L8.attn.11", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 2 ], [ 37, 4 ], [ 42, 2 ], [ 68, 1 ], [ 70, 1 ], [ 81, 1 ], [ 84, 1 ], [ 92, 26 ], [ 120, 233 ], [ 419, 250 ], [ 702, 15 ], [ 762, 33 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 15 ], [ 878, 1 ], [ 881, 7 ], [ 929, 4 ], [ 966, 16 ], [ 983, 5 ], [ 993, 27 ], [ 1021, 2 ], [ 1024, 17 ], [ 1042, 4 ], [ 1047, 4 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 17 ], [ 1082, 29 ], [ 1112, 1 ], [ 1187, 3 ], [ 1561, 17 ], [ 1586, 1 ], [ 1636, 2 ], [ 1639, 2 ], [ 1642, 16 ], [ 1677, 1 ], [ 1698, 1 ], [ 1701, 1 ], [ 1703, 1 ], [ 1706, 1 ], [ 1709, 3 ], [ 1750, 13 ], [ 1800, 4 ], [ 1845, 32 ], [ 1943, 33 ], [ 2045, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 32, "index": 8, "mlp_operations": [ { "hidden_slots": [ [ 1, 3 ] ], "id": "L8.mlp.0", "node": "l:169", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1690, 1 ], [ 477, 3 ] ], "source_columns_b": null, "target_columns": [ [ 480, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 4, 14 ] ], "id": "L8.mlp.1", "node": "l:608", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 36, 1 ] ], "source_columns_b": null, "target_columns": [ [ 483, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 18, 33 ] ], "id": "L8.mlp.2", "node": "l:594", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 44, 1 ], [ 46, 4 ], [ 51, 4 ], [ 58, 1 ], [ 61, 2 ], [ 66, 2 ], [ 69, 1 ], [ 71, 10 ], [ 83, 1 ], [ 85, 7 ] ], "source_columns_b": null, "target_columns": [ [ 516, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 51, 33 ] ], "id": "L8.mlp.3", "node": "l:642", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1844, 1 ], [ 180, 33 ] ], "source_columns_b": null, "target_columns": [ [ 549, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 84, 33 ] ], "id": "L8.mlp.4", "node": "l:702", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1332, 1 ], [ 213, 33 ] ], "source_columns_b": null, "target_columns": [ [ 582, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 117, 33 ] ], "id": "L8.mlp.5", "node": "l:751", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1461, 1 ], [ 246, 33 ] ], "source_columns_b": null, "target_columns": [ [ 615, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 150, 33 ] ], "id": "L8.mlp.6", "node": "l:799", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1081, 1 ], [ 279, 33 ] ], "source_columns_b": null, "target_columns": [ [ 648, 21 ], [ 702, 12 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 183, 33 ] ], "id": "L8.mlp.7", "node": "l:920", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1492, 1 ], [ 312, 33 ] ], "source_columns_b": null, "target_columns": [ [ 714, 3 ], [ 762, 30 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 216, 33 ] ], "id": "L8.mlp.8", "node": "l:968", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1369, 1 ], [ 345, 8 ], [ 419, 25 ] ], "source_columns_b": null, "target_columns": [ [ 792, 3 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 6 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 249, 2 ] ], "id": "L8.mlp.9", "node": "l:526", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 868, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 251, 66 ] ], "id": "L8.mlp.10", "node": "l:896", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 63, 1 ], [ 1578, 8 ], [ 1587, 24 ], [ 65, 1 ], [ 1611, 24 ], [ 1641, 1 ], [ 1700, 1 ], [ 1808, 1 ], [ 1810, 1 ], [ 1812, 1 ], [ 1838, 2 ], [ 1842, 1 ], [ 669, 33 ] ], "source_columns_b": null, "target_columns": [ [ 869, 8 ], [ 878, 1 ], [ 881, 7 ], [ 929, 4 ], [ 966, 13 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L8.mlp.11", "node": "l:185", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 979, 3 ], [ 983, 5 ], [ 993, 25 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.12", "node": "l:194", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1018, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.13", "node": "l:197", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1019, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.14", "node": "l:227", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1021, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.15", "node": "l:91", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1022, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.16", "node": "l:94", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1024, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.17", "node": "l:144", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1025, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.18", "node": "l:145", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1026, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.19", "node": "l:147", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1027, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.20", "node": "l:533", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1028, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.21", "node": "l:539", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1029, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.22", "node": "l:957", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1030, 11 ], [ 1042, 4 ], [ 1047, 4 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 4 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.23", "node": "l:1029", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1068, 13 ], [ 1082, 20 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.24", "node": "l:303", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1102, 9 ], [ 1112, 1 ], [ 1187, 3 ], [ 1223, 20 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.25", "node": "l:174", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 477, 3 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 317, 66 ] ], "id": "L8.mlp.26", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1190, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1190, 33 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 383, 66 ] ], "id": "L8.mlp.27", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1910, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1910, 33 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 449, 2 ] ], "id": "L8.mlp.28", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1844, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1844, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 451, 2 ] ], "id": "L8.mlp.29", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1332, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1332, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 453, 2 ] ], "id": "L8.mlp.30", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1461, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1461, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 455, 2 ] ], "id": "L8.mlp.31", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1492, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1492, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 457, 2 ] ], "id": "L8.mlp.32", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1369, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1369, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:174", "l:170", "l:169", "l:168", "l:185", "l:184", "l:198", "l:194", "l:197", "l:193", "l:196", "l:231", "l:227", "l:226", "l:95", "l:91", "l:94", "l:90", "l:93", "l:144", "l:145", "l:147", "l:143", "l:146", "l:526", "l:533", "l:539", "l:957", "l:1029", "l:302", "l:594", "l:607", "l:648", "l:708", "l:757", "l:805", "l:896", "l:926", "l:956", "l:974", "l:998", "l:1028", "l:643", "l:642", "l:703", "l:702", "l:752", "l:751", "l:800", "l:799", "l:891", "l:921", "l:920", "l:969", "l:968", "l:993", "l:608", "l:303" ] }, { "active_attention_heads": 15, "active_mlp_neurons": 318, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L9.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:167", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 1, "role": "transport", "source_columns": [ [ 480, 3 ], [ 118, 1 ] ], "source_columns_b": null, "target_columns": [ [ 64, 1 ], [ 1691, 3 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L9.attn.1", "key_source_columns": [ [ 41, 1 ], [ 82, 1 ] ], "node": "l:150", "query_source_columns": [ [ 1179, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1183, 1 ] ], "source_columns_b": null, "target_columns": [ [ 34, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L9.attn.2", "key_source_columns": [ [ 41, 1 ], [ 1019, 1 ] ], "node": "l:192", "query_source_columns": [ [ 1018, 1 ], [ 119, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 34, 2 ], [ 37, 4 ], [ 42, 2 ], [ 50, 1 ], [ 68, 1 ], [ 70, 1 ], [ 81, 1 ], [ 84, 1 ], [ 92, 20 ] ], "source_columns_b": null, "target_columns": [ [ 35, 6 ], [ 42, 3 ], [ 46, 9 ], [ 58, 1 ], [ 61, 3 ], [ 65, 11 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L9.attn.3", "key_source_columns": [ [ 41, 1 ], [ 1182, 1 ] ], "node": "l:225", "query_source_columns": [ [ 1021, 1 ], [ 1181, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 112, 6 ], [ 120, 27 ] ], "source_columns_b": null, "target_columns": [ [ 76, 16 ], [ 109, 6 ], [ 119, 1 ], [ 133, 7 ], [ 143, 3 ] ], "type": "compute_attn" }, { "heads": [ 4, 1 ], "id": "L9.attn.4", "key_source_columns": [ [ 41, 1 ], [ 1024, 1 ] ], "node": "l:89", "query_source_columns": [ [ 1022, 1 ], [ 1804, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 147, 33 ] ], "source_columns_b": null, "target_columns": [ [ 146, 6 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 174, 1 ], [ 177, 8 ], [ 186, 6 ], [ 195, 1 ], [ 197, 1 ], [ 222, 1 ] ], "type": "compute_attn" }, { "heads": [ 5, 1 ], "id": "L9.attn.5", "key_source_columns": [ [ 1028, 1 ] ], "node": "l:532", "query_source_columns": [ [ 1028, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 172, 1 ], [ 223, 12 ], [ 237, 4 ], [ 247, 16 ] ], "type": "compute_attn" }, { "heads": [ 6, 1 ], "id": "L9.attn.6", "key_source_columns": [ [ 1029, 1 ] ], "node": "l:538", "query_source_columns": [ [ 1029, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 33 ] ], "source_columns_b": null, "target_columns": [ [ 263, 33 ] ], "type": "compute_attn" }, { "heads": [ 7, 8 ], "id": "L9.attn.7", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 7 ], [ 42, 3 ], [ 46, 9 ], [ 58, 1 ], [ 61, 3 ], [ 65, 53 ], [ 119, 234 ], [ 419, 25 ], [ 477, 3 ], [ 669, 33 ], [ 1018, 2 ], [ 1021, 2 ], [ 1024, 1 ], [ 1028, 2 ], [ 1081, 1 ], [ 1179, 5 ], [ 1578, 8 ], [ 1587, 48 ], [ 1641, 1 ], [ 1700, 1 ], [ 1804, 1 ], [ 1808, 1 ], [ 1810, 1 ], [ 1812, 1 ], [ 1838, 2 ], [ 1842, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 15, "index": 9, "mlp_operations": [ { "hidden_slots": [ [ 1, 66 ] ], "id": "L9.mlp.0", "node": "l:183", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1690, 1 ], [ 1, 33 ], [ 979, 3 ], [ 983, 5 ], [ 993, 25 ] ], "source_columns_b": null, "target_columns": [ [ 296, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 67, 13 ] ], "id": "L9.mlp.1", "node": "l:241", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 34, 1 ] ], "source_columns_b": null, "target_columns": [ [ 329, 1 ], [ 332, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 80, 9 ] ], "id": "L9.mlp.2", "node": "l:207", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 34, 1 ] ], "source_columns_b": null, "target_columns": [ [ 335, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 89, 17 ] ], "id": "L9.mlp.3", "node": "l:149", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 34, 1 ] ], "source_columns_b": null, "target_columns": [ [ 339, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 106, 1 ] ], "id": "L9.mlp.4", "node": "l:531", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 172, 1 ], [ 223, 12 ], [ 237, 4 ], [ 247, 16 ] ], "source_columns_b": null, "target_columns": [ [ 344, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 107, 2 ] ], "id": "L9.mlp.5", "node": "l:530", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 868, 1 ] ], "source_columns_b": null, "target_columns": [ [ 345, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 109, 1 ] ], "id": "L9.mlp.6", "node": "l:537", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 263, 33 ] ], "source_columns_b": null, "target_columns": [ [ 346, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 110, 33 ] ], "id": "L9.mlp.7", "node": "l:301", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 1102, 9 ], [ 1112, 1 ], [ 1187, 3 ], [ 1223, 20 ] ], "source_columns_b": null, "target_columns": [ [ 92, 2 ], [ 343, 1 ], [ 347, 6 ], [ 419, 20 ], [ 440, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 143, 33 ] ], "id": "L9.mlp.8", "node": "l:606", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 483, 33 ] ], "source_columns_b": null, "target_columns": [ [ 94, 15 ], [ 115, 3 ], [ 120, 10 ], [ 241, 1 ], [ 330, 1 ], [ 477, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 176, 33 ] ], "id": "L9.mlp.9", "node": "l:955", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 1030, 11 ], [ 1042, 4 ], [ 1047, 4 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 4 ] ], "source_columns_b": null, "target_columns": [ [ 130, 3 ], [ 669, 30 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 209, 33 ] ], "id": "L9.mlp.10", "node": "l:1027", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 1068, 13 ], [ 1082, 20 ] ], "source_columns_b": null, "target_columns": [ [ 140, 3 ], [ 152, 6 ], [ 165, 3 ], [ 169, 2 ], [ 173, 1 ], [ 175, 2 ], [ 185, 1 ], [ 192, 3 ], [ 196, 1 ], [ 198, 4 ], [ 203, 1 ], [ 331, 1 ], [ 699, 3 ], [ 1018, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 242, 33 ] ], "id": "L9.mlp.11", "node": "l:890", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1357, 1 ], [ 869, 8 ], [ 878, 1 ], [ 881, 7 ], [ 929, 4 ], [ 966, 13 ] ], "source_columns_b": null, "target_columns": [ [ 202, 1 ], [ 204, 8 ], [ 213, 9 ], [ 235, 2 ], [ 1021, 2 ], [ 1024, 1 ], [ 1028, 2 ], [ 1081, 1 ], [ 1179, 5 ], [ 1190, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 275, 33 ] ], "id": "L9.mlp.12", "node": "l:992", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1382, 1 ], [ 444, 33 ] ], "source_columns_b": null, "target_columns": [ [ 242, 2 ], [ 1192, 31 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 308, 2 ] ], "id": "L9.mlp.13", "node": "l:393", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 35, 6 ], [ 42, 3 ], [ 46, 9 ], [ 58, 1 ], [ 61, 3 ], [ 65, 11 ] ], "source_columns_b": null, "target_columns": [ [ 244, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 310, 2 ] ], "id": "L9.mlp.14", "node": "l:352", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 76, 16 ], [ 109, 6 ], [ 119, 1 ], [ 133, 7 ], [ 143, 3 ] ], "source_columns_b": null, "target_columns": [ [ 246, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 312, 2 ] ], "id": "L9.mlp.15", "node": "l:312", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 146, 6 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 174, 1 ], [ 177, 8 ], [ 186, 6 ], [ 195, 1 ], [ 197, 1 ], [ 222, 1 ] ], "source_columns_b": null, "target_columns": [ [ 245, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 314, 2 ] ], "id": "L9.mlp.16", "node": "l:274", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 35, 6 ], [ 42, 3 ], [ 46, 9 ], [ 58, 1 ], [ 61, 3 ], [ 65, 11 ] ], "source_columns_b": null, "target_columns": [ [ 212, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 316, 2 ] ], "id": "L9.mlp.17", "node": "l:224", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 76, 16 ], [ 109, 6 ], [ 119, 1 ], [ 133, 7 ], [ 143, 3 ] ], "source_columns_b": null, "target_columns": [ [ 1243, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L9.mlp.18", "node": "l:210", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1244, 33 ] ], "type": "compute_literal_value" } ], "newly_computed": [ "l:167", "l:183", "l:150", "l:192", "l:225", "l:89", "l:274", "l:224", "l:532", "l:531", "l:530", "l:529", "l:393", "l:352", "l:312", "l:538", "l:537", "l:209", "l:301", "l:606", "l:955", "l:951", "l:1027", "l:1023", "l:890", "l:992", "l:210", "l:149", "l:241", "l:207" ] }, { "active_attention_heads": 10, "active_mlp_neurons": 168, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L10.attn.0", "key_source_columns": [ [ 64, 1 ], [ 1691, 3 ] ], "node": "l:206", "query_source_columns": [ [ 335, 4 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 296, 33 ] ], "source_columns_b": null, "target_columns": [ [ 34, 1 ], [ 64, 1 ], [ 172, 1 ], [ 223, 12 ], [ 254, 17 ], [ 328, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L10.attn.1", "key_source_columns": [ [ 64, 1 ], [ 1691, 3 ] ], "node": "l:240", "query_source_columns": [ [ 329, 1 ], [ 332, 3 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 296, 33 ] ], "source_columns_b": null, "target_columns": [ [ 271, 33 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L10.attn.2", "key_source_columns": [ [ 64, 1 ], [ 1691, 3 ] ], "node": "l:148", "query_source_columns": [ [ 339, 4 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 296, 33 ] ], "source_columns_b": null, "target_columns": [ [ 304, 23 ], [ 332, 3 ], [ 336, 7 ] ], "type": "compute_attn" }, { "heads": [ 3, 7 ], "id": "L10.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1, 34 ], [ 64, 1 ], [ 118, 1 ], [ 172, 1 ], [ 223, 12 ], [ 237, 4 ], [ 247, 83 ], [ 332, 11 ], [ 444, 33 ], [ 480, 36 ], [ 869, 8 ], [ 878, 1 ], [ 881, 7 ], [ 929, 4 ], [ 966, 16 ], [ 983, 5 ], [ 993, 25 ], [ 1030, 11 ], [ 1042, 4 ], [ 1047, 4 ], [ 1052, 9 ], [ 1062, 1 ], [ 1064, 17 ], [ 1082, 29 ], [ 1112, 1 ], [ 1187, 3 ], [ 1223, 20 ], [ 1357, 1 ], [ 1382, 1 ], [ 1690, 4 ] ], "type": "cancel" } ], "emitted_attention_heads": 10, "index": 10, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L10.mlp.0", "node": "l:528", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 345, 1 ], [ 344, 1 ] ], "source_columns_b": null, "target_columns": [ [ 329, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 33 ] ], "id": "L10.mlp.1", "node": "l:208", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 1244, 33 ] ], "source_columns_b": null, "target_columns": [ [ 444, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 36, 66 ] ], "id": "L10.mlp.2", "node": "l:950", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1877, 33 ], [ 130, 3 ], [ 669, 30 ], [ 933, 33 ] ], "source_columns_b": null, "target_columns": [ [ 480, 33 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 102, 66 ] ], "id": "L10.mlp.3", "node": "l:1022", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1113, 33 ], [ 140, 3 ], [ 152, 6 ], [ 165, 3 ], [ 169, 2 ], [ 173, 1 ], [ 175, 2 ], [ 185, 1 ], [ 192, 3 ], [ 196, 1 ], [ 198, 4 ], [ 203, 1 ], [ 331, 1 ], [ 699, 3 ], [ 1018, 2 ], [ 1146, 33 ] ], "source_columns_b": null, "target_columns": [ [ 118, 1 ], [ 513, 3 ], [ 869, 8 ], [ 878, 1 ], [ 881, 7 ], [ 930, 1 ], [ 966, 6 ], [ 975, 4 ], [ 1008, 1 ], [ 1012, 1 ] ], "type": "compute_linear_bypass" } ], "newly_computed": [ "l:206", "l:240", "l:148", "l:528", "l:208", "l:950", "l:1022", "l:945", "l:1017" ] }, { "active_attention_heads": 9, "active_mlp_neurons": 1889, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L11.attn.0", "key_source_columns": [ [ 41, 1 ], [ 1186, 1 ] ], "node": "l:200", "query_source_columns": [ [ 1184, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 34, 1 ], [ 64, 1 ], [ 172, 1 ], [ 223, 12 ], [ 254, 17 ], [ 328, 1 ] ], "source_columns_b": null, "target_columns": [ [ 34, 1 ], [ 41, 1 ], [ 172, 2 ], [ 196, 1 ], [ 199, 1 ], [ 223, 12 ], [ 254, 13 ], [ 331, 1 ], [ 1150, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L11.attn.1", "key_source_columns": [ [ 41, 1 ], [ 1725, 1 ] ], "node": "l:234", "query_source_columns": [ [ 1635, 1 ], [ 1724, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 271, 33 ] ], "source_columns_b": null, "target_columns": [ [ 267, 33 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L11.attn.2", "key_source_columns": [ [ 41, 1 ], [ 1027, 1 ] ], "node": "l:142", "query_source_columns": [ [ 1025, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 304, 23 ], [ 332, 3 ], [ 336, 7 ] ], "source_columns_b": null, "target_columns": [ [ 64, 1 ], [ 300, 21 ], [ 322, 1 ], [ 324, 3 ], [ 328, 1 ], [ 332, 3 ], [ 336, 3 ] ], "type": "compute_attn" }, { "heads": [ 3, 6 ], "id": "L11.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 1 ], [ 41, 1 ], [ 64, 1 ], [ 130, 3 ], [ 140, 3 ], [ 152, 6 ], [ 165, 3 ], [ 169, 2 ], [ 172, 2 ], [ 175, 2 ], [ 185, 1 ], [ 192, 3 ], [ 196, 1 ], [ 198, 4 ], [ 203, 1 ], [ 223, 12 ], [ 254, 73 ], [ 328, 1 ], [ 331, 4 ], [ 336, 7 ], [ 344, 2 ], [ 669, 33 ], [ 933, 33 ], [ 1018, 2 ], [ 1025, 3 ], [ 1113, 66 ], [ 1184, 3 ], [ 1244, 33 ], [ 1635, 1 ], [ 1724, 2 ], [ 1877, 33 ] ], "type": "cancel" } ], "emitted_attention_heads": 9, "index": 11, "mlp_operations": [ { "hidden_slots": [ [ 1, 100 ] ], "id": "L11.mlp.0", "node": "l:365", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 6 ], [ 42, 3 ], [ 46, 9 ], [ 58, 1 ], [ 61, 3 ], [ 65, 11 ], [ 34, 1 ], [ 41, 1 ], [ 172, 2 ], [ 196, 1 ], [ 199, 1 ], [ 223, 12 ], [ 254, 13 ], [ 331, 1 ], [ 1150, 1 ] ], "source_columns_b": null, "target_columns": [ [ 339, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 101, 100 ] ], "id": "L11.mlp.1", "node": "l:323", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 6 ], [ 42, 3 ], [ 46, 9 ], [ 58, 1 ], [ 61, 3 ], [ 65, 11 ], [ 267, 33 ] ], "source_columns_b": null, "target_columns": [ [ 341, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 201, 100 ] ], "id": "L11.mlp.2", "node": "l:287", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 6 ], [ 42, 3 ], [ 46, 9 ], [ 58, 1 ], [ 61, 3 ], [ 65, 11 ], [ 64, 1 ], [ 300, 21 ], [ 322, 1 ], [ 324, 3 ], [ 328, 1 ], [ 332, 3 ], [ 336, 3 ] ], "source_columns_b": null, "target_columns": [ [ 344, 1 ], [ 1025, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 301, 100 ] ], "id": "L11.mlp.3", "node": "l:325", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 76, 16 ], [ 109, 6 ], [ 119, 1 ], [ 133, 7 ], [ 143, 3 ], [ 34, 1 ], [ 41, 1 ], [ 172, 2 ], [ 196, 1 ], [ 199, 1 ], [ 223, 12 ], [ 254, 13 ], [ 331, 1 ], [ 1150, 1 ] ], "source_columns_b": null, "target_columns": [ [ 185, 1 ], [ 345, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 401, 100 ] ], "id": "L11.mlp.4", "node": "l:289", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 76, 16 ], [ 109, 6 ], [ 119, 1 ], [ 133, 7 ], [ 143, 3 ], [ 267, 33 ] ], "source_columns_b": null, "target_columns": [ [ 130, 1 ], [ 200, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 501, 100 ] ], "id": "L11.mlp.5", "node": "l:254", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 76, 16 ], [ 109, 6 ], [ 119, 1 ], [ 133, 7 ], [ 143, 3 ], [ 64, 1 ], [ 300, 21 ], [ 322, 1 ], [ 324, 3 ], [ 328, 1 ], [ 332, 3 ], [ 336, 3 ] ], "source_columns_b": null, "target_columns": [ [ 131, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 601, 100 ] ], "id": "L11.mlp.6", "node": "l:291", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 146, 6 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 174, 1 ], [ 177, 8 ], [ 186, 6 ], [ 195, 1 ], [ 197, 1 ], [ 222, 1 ], [ 34, 1 ], [ 41, 1 ], [ 172, 2 ], [ 196, 1 ], [ 199, 1 ], [ 223, 12 ], [ 254, 13 ], [ 331, 1 ], [ 1150, 1 ] ], "source_columns_b": null, "target_columns": [ [ 669, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 701, 100 ] ], "id": "L11.mlp.7", "node": "l:256", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 146, 6 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 174, 1 ], [ 177, 8 ], [ 186, 6 ], [ 195, 1 ], [ 197, 1 ], [ 222, 1 ], [ 267, 33 ] ], "source_columns_b": null, "target_columns": [ [ 671, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 801, 100 ] ], "id": "L11.mlp.8", "node": "l:258", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 146, 6 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 174, 1 ], [ 177, 8 ], [ 186, 6 ], [ 195, 1 ], [ 197, 1 ], [ 222, 1 ], [ 64, 1 ], [ 300, 21 ], [ 322, 1 ], [ 324, 3 ], [ 328, 1 ], [ 332, 3 ], [ 336, 3 ] ], "source_columns_b": null, "target_columns": [ [ 673, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 901, 300 ] ], "id": "L11.mlp.9", "node": "l:189", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 55, 3 ], [ 35, 6 ], [ 42, 3 ], [ 46, 9 ], [ 58, 1 ], [ 61, 3 ], [ 65, 11 ], [ 34, 1 ], [ 41, 1 ], [ 172, 2 ], [ 196, 1 ], [ 199, 1 ], [ 223, 12 ], [ 254, 13 ], [ 331, 1 ], [ 1150, 1 ] ], "source_columns_b": null, "target_columns": [ [ 675, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1201, 300 ] ], "id": "L11.mlp.10", "node": "l:246", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 55, 3 ], [ 76, 16 ], [ 109, 6 ], [ 119, 1 ], [ 133, 7 ], [ 143, 3 ], [ 267, 33 ] ], "source_columns_b": null, "target_columns": [ [ 678, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1501, 300 ] ], "id": "L11.mlp.11", "node": "l:279", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 55, 3 ], [ 146, 6 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 174, 1 ], [ 177, 8 ], [ 186, 6 ], [ 195, 1 ], [ 197, 1 ], [ 222, 1 ], [ 64, 1 ], [ 300, 21 ], [ 322, 1 ], [ 324, 3 ], [ 328, 1 ], [ 332, 3 ], [ 336, 3 ] ], "source_columns_b": null, "target_columns": [ [ 681, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1801, 33 ] ], "id": "L11.mlp.12", "node": "l:944", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1111, 1 ], [ 480, 33 ] ], "source_columns_b": null, "target_columns": [ [ 140, 3 ], [ 152, 6 ], [ 165, 3 ], [ 169, 1 ], [ 193, 2 ], [ 684, 18 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1834, 33 ] ], "id": "L11.mlp.13", "node": "l:1016", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1386, 1 ], [ 118, 1 ], [ 513, 3 ], [ 869, 8 ], [ 878, 1 ], [ 881, 7 ], [ 930, 1 ], [ 966, 6 ], [ 975, 4 ], [ 1008, 1 ], [ 1012, 1 ] ], "source_columns_b": null, "target_columns": [ [ 170, 1 ], [ 175, 2 ], [ 929, 1 ], [ 931, 28 ], [ 999, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1867, 2 ] ], "id": "L11.mlp.14", "node": "l:394", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 34, 1 ], [ 41, 1 ], [ 172, 2 ], [ 196, 1 ], [ 199, 1 ], [ 223, 12 ], [ 254, 13 ], [ 331, 1 ], [ 1150, 1 ] ], "source_columns_b": null, "target_columns": [ [ 959, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 1869, 2 ] ], "id": "L11.mlp.15", "node": "l:353", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 267, 33 ] ], "source_columns_b": null, "target_columns": [ [ 960, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 1871, 2 ] ], "id": "L11.mlp.16", "node": "l:313", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 64, 1 ], [ 300, 21 ], [ 322, 1 ], [ 324, 3 ], [ 328, 1 ], [ 332, 3 ], [ 336, 3 ] ], "source_columns_b": null, "target_columns": [ [ 961, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 1873, 2 ] ], "id": "L11.mlp.17", "node": "l:275", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 34, 1 ], [ 41, 1 ], [ 172, 2 ], [ 196, 1 ], [ 199, 1 ], [ 223, 12 ], [ 254, 13 ], [ 331, 1 ], [ 1150, 1 ] ], "source_columns_b": null, "target_columns": [ [ 962, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 1875, 2 ] ], "id": "L11.mlp.18", "node": "l:233", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 267, 33 ] ], "source_columns_b": null, "target_columns": [ [ 963, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 1877, 2 ] ], "id": "L11.mlp.19", "node": "l:88", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 146, 6 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 174, 1 ], [ 177, 8 ], [ 186, 6 ], [ 195, 1 ], [ 197, 1 ], [ 222, 1 ] ], "source_columns_b": null, "target_columns": [ [ 201, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 1879, 2 ] ], "id": "L11.mlp.20", "node": "l:141", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 64, 1 ], [ 300, 21 ], [ 322, 1 ], [ 324, 3 ], [ 328, 1 ], [ 332, 3 ], [ 336, 3 ] ], "source_columns_b": null, "target_columns": [ [ 192, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 1881, 2 ] ], "id": "L11.mlp.21", "node": "l:527", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 329, 1 ] ], "source_columns_b": null, "target_columns": [ [ 964, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 1883, 6 ] ], "id": "L11.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 55, 3 ] ], "source_columns_b": null, "target_columns": [ [ 55, 3 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:200", "l:234", "l:142", "l:88", "l:275", "l:233", "l:141", "l:527", "l:394", "l:353", "l:313", "l:190", "l:189", "l:247", "l:246", "l:280", "l:279", "l:309", "l:316", "l:347", "l:356", "l:388", "l:397", "l:425", "l:432", "l:461", "l:468", "l:499", "l:506", "l:366", "l:365", "l:324", "l:323", "l:288", "l:287", "l:326", "l:325", "l:290", "l:289", "l:255", "l:254", "l:292", "l:291", "l:257", "l:256", "l:259", "l:258", "l:253", "l:286", "l:322", "l:364", "l:944", "l:1016" ] }, { "active_attention_heads": 13, "active_mlp_neurons": 218, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L12.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:223", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 963, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1243, 1 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L12.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:273", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 962, 1 ] ], "source_columns_b": null, "target_columns": [ [ 212, 1 ] ], "type": "add_into" }, { "heads": [ 2, 1 ], "id": "L12.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:87", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 192, 1 ] ], "source_columns_b": null, "target_columns": [ [ 201, 1 ] ], "type": "add_into" }, { "heads": [ 3, 1 ], "id": "L12.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": "l:525", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 964, 1 ] ], "source_columns_b": null, "target_columns": [ [ 868, 1 ] ], "type": "add_into" }, { "heads": [ 4, 1 ], "id": "L12.attn.4", "key_source_columns": [ [ 0, 1 ] ], "node": "l:621", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 673, 2 ] ], "source_columns_b": null, "target_columns": [ [ 52, 1 ] ], "type": "compute_linear" }, { "heads": [ 5, 1 ], "id": "L12.attn.5", "key_source_columns": [ [ 0, 1 ] ], "node": "l:188", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 675, 3 ] ], "source_columns_b": null, "target_columns": [ [ 50, 2 ], [ 53, 5 ], [ 72, 12 ], [ 143, 9 ], [ 158, 3 ], [ 229, 1 ], [ 332, 1 ] ], "type": "compute_linear" }, { "heads": [ 6, 1 ], "id": "L12.attn.6", "key_source_columns": [ [ 0, 1 ] ], "node": "l:245", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 678, 3 ] ], "source_columns_b": null, "target_columns": [ [ 34, 1 ], [ 36, 1 ], [ 39, 1 ], [ 41, 1 ], [ 43, 2 ], [ 49, 1 ], [ 161, 4 ], [ 168, 1 ], [ 171, 1 ], [ 177, 8 ], [ 195, 3 ], [ 199, 1 ], [ 222, 6 ], [ 231, 1 ], [ 255, 1 ] ], "type": "compute_linear" }, { "heads": [ 7, 1 ], "id": "L12.attn.7", "key_source_columns": [ [ 0, 1 ] ], "node": "l:278", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 681, 3 ] ], "source_columns_b": null, "target_columns": [ [ 67, 1 ], [ 173, 1 ], [ 230, 1 ], [ 254, 1 ], [ 256, 28 ], [ 1150, 1 ] ], "type": "compute_linear" }, { "heads": [ 8, 5 ], "id": "L12.attn.8", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 11 ], [ 46, 9 ], [ 58, 1 ], [ 61, 31 ], [ 109, 6 ], [ 118, 2 ], [ 133, 7 ], [ 143, 9 ], [ 158, 7 ], [ 168, 1 ], [ 171, 4 ], [ 177, 8 ], [ 186, 6 ], [ 195, 3 ], [ 199, 1 ], [ 222, 13 ], [ 254, 67 ], [ 322, 1 ], [ 324, 3 ], [ 328, 2 ], [ 331, 4 ], [ 336, 3 ], [ 480, 36 ], [ 675, 9 ], [ 869, 8 ], [ 878, 1 ], [ 881, 7 ], [ 930, 1 ], [ 966, 6 ], [ 975, 4 ], [ 1008, 1 ], [ 1012, 1 ], [ 1111, 1 ], [ 1150, 1 ], [ 1386, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 13, "index": 12, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L12.mlp.0", "node": "l:355", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 988, 1 ], [ 960, 1 ], [ 246, 1 ] ], "source_columns_b": null, "target_columns": [ [ 174, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L12.mlp.1", "node": "l:396", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 989, 1 ], [ 959, 1 ], [ 244, 1 ] ], "source_columns_b": null, "target_columns": [ [ 286, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 2 ] ], "id": "L12.mlp.2", "node": "l:431", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 990, 1 ], [ 959, 1 ], [ 244, 1 ] ], "source_columns_b": null, "target_columns": [ [ 287, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 7, 2 ] ], "id": "L12.mlp.3", "node": "l:467", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 991, 1 ], [ 960, 1 ], [ 246, 1 ] ], "source_columns_b": null, "target_columns": [ [ 288, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 9, 2 ] ], "id": "L12.mlp.4", "node": "l:315", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 45, 1 ], [ 961, 1 ], [ 245, 1 ] ], "source_columns_b": null, "target_columns": [ [ 289, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 11, 2 ] ], "id": "L12.mlp.5", "node": "l:346", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 988, 1 ], [ 246, 1 ], [ 960, 1 ] ], "source_columns_b": null, "target_columns": [ [ 290, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 13, 2 ] ], "id": "L12.mlp.6", "node": "l:387", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 989, 1 ], [ 244, 1 ], [ 959, 1 ] ], "source_columns_b": null, "target_columns": [ [ 291, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 15, 2 ] ], "id": "L12.mlp.7", "node": "l:424", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 990, 1 ], [ 244, 1 ], [ 959, 1 ] ], "source_columns_b": null, "target_columns": [ [ 58, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 17, 2 ] ], "id": "L12.mlp.8", "node": "l:460", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 991, 1 ], [ 246, 1 ], [ 960, 1 ] ], "source_columns_b": null, "target_columns": [ [ 293, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 19, 2 ] ], "id": "L12.mlp.9", "node": "l:505", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 992, 1 ], [ 961, 1 ], [ 245, 1 ] ], "source_columns_b": null, "target_columns": [ [ 294, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 21, 2 ] ], "id": "L12.mlp.10", "node": "l:308", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 45, 1 ], [ 245, 1 ], [ 961, 1 ] ], "source_columns_b": null, "target_columns": [ [ 295, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 23, 2 ] ], "id": "L12.mlp.11", "node": "l:498", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 992, 1 ], [ 245, 1 ], [ 961, 1 ] ], "source_columns_b": null, "target_columns": [ [ 296, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 25, 18 ] ], "id": "L12.mlp.12", "node": "l:620", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 52, 1 ] ], "source_columns_b": null, "target_columns": [ [ 61, 5 ], [ 297, 10 ], [ 308, 4 ], [ 313, 4 ], [ 318, 3 ], [ 322, 1 ], [ 324, 3 ], [ 328, 1 ], [ 333, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 43, 14 ] ], "id": "L12.mlp.13", "node": "l:524", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 868, 1 ] ], "source_columns_b": null, "target_columns": [ [ 46, 1 ], [ 68, 1 ], [ 70, 2 ], [ 84, 8 ], [ 329, 1 ], [ 336, 3 ], [ 480, 16 ], [ 882, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 57, 36 ] ], "id": "L12.mlp.14", "node": "l:454", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 201, 1 ] ], "source_columns_b": null, "target_columns": [ [ 37, 1 ], [ 69, 1 ], [ 109, 6 ], [ 118, 2 ], [ 186, 2 ], [ 232, 1 ], [ 496, 18 ], [ 515, 1 ], [ 930, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 93, 2 ] ], "id": "L12.mlp.15", "node": "l:86", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 201, 1 ] ], "source_columns_b": null, "target_columns": [ [ 38, 1 ], [ 40, 1 ], [ 66, 1 ], [ 133, 7 ], [ 188, 4 ], [ 233, 2 ], [ 331, 1 ], [ 675, 9 ], [ 966, 6 ], [ 1008, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 95, 33 ] ], "id": "L12.mlp.16", "node": "l:186", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 50, 2 ], [ 53, 5 ], [ 72, 12 ], [ 143, 9 ], [ 158, 3 ], [ 229, 1 ], [ 332, 1 ] ], "source_columns_b": null, "target_columns": [ [ 35, 1 ], [ 228, 1 ], [ 327, 1 ], [ 869, 8 ], [ 878, 1 ], [ 883, 5 ], [ 965, 1 ], [ 972, 10 ], [ 984, 2 ], [ 1006, 2 ], [ 1009, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 128, 33 ] ], "id": "L12.mlp.17", "node": "l:243", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 34, 1 ], [ 36, 1 ], [ 39, 1 ], [ 41, 1 ], [ 43, 2 ], [ 49, 1 ], [ 161, 4 ], [ 168, 1 ], [ 171, 1 ], [ 177, 8 ], [ 195, 3 ], [ 199, 1 ], [ 222, 6 ], [ 231, 1 ], [ 255, 1 ] ], "source_columns_b": null, "target_columns": [ [ 198, 1 ], [ 203, 1 ], [ 983, 1 ], [ 986, 2 ], [ 993, 6 ], [ 1000, 6 ], [ 1010, 9 ], [ 1032, 6 ], [ 1042, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 161, 33 ] ], "id": "L12.mlp.18", "node": "l:276", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 67, 1 ], [ 173, 1 ], [ 230, 1 ], [ 254, 1 ], [ 256, 28 ], [ 1150, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1026, 1 ], [ 1038, 3 ], [ 1043, 3 ], [ 1047, 2 ], [ 1052, 8 ], [ 1068, 13 ], [ 1082, 2 ], [ 1087, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 194, 2 ] ], "id": "L12.mlp.19", "node": "l:252", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 131, 2 ], [ 671, 4 ] ], "source_columns_b": null, "target_columns": [ [ 1084, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 196, 2 ] ], "id": "L12.mlp.20", "node": "l:285", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 344, 1 ], [ 1025, 1 ], [ 130, 1 ], [ 200, 1 ], [ 131, 2 ], [ 669, 4 ] ], "source_columns_b": null, "target_columns": [ [ 1085, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 198, 2 ] ], "id": "L12.mlp.21", "node": "l:321", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 341, 2 ], [ 344, 1 ], [ 1025, 1 ], [ 185, 1 ], [ 345, 1 ], [ 130, 1 ], [ 200, 1 ], [ 669, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1086, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 200, 2 ] ], "id": "L12.mlp.22", "node": "l:363", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 339, 4 ], [ 185, 1 ], [ 345, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1030, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 202, 2 ] ], "id": "L12.mlp.23", "node": "l:403", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 339, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1088, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 204, 2 ] ], "id": "L12.mlp.24", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 244, 1 ] ], "source_columns_b": null, "target_columns": [ [ 244, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 206, 2 ] ], "id": "L12.mlp.25", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 246, 1 ] ], "source_columns_b": null, "target_columns": [ [ 246, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 208, 2 ] ], "id": "L12.mlp.26", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 959, 1 ] ], "source_columns_b": null, "target_columns": [ [ 959, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 210, 2 ] ], "id": "L12.mlp.27", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 988, 1 ] ], "source_columns_b": null, "target_columns": [ [ 988, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 212, 2 ] ], "id": "L12.mlp.28", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 989, 1 ] ], "source_columns_b": null, "target_columns": [ [ 989, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 214, 2 ] ], "id": "L12.mlp.29", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 991, 1 ] ], "source_columns_b": null, "target_columns": [ [ 991, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 216, 2 ] ], "id": "L12.mlp.30", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 992, 1 ] ], "source_columns_b": null, "target_columns": [ [ 992, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:87", "l:223", "l:273", "l:525", "l:188", "l:245", "l:278", "l:308", "l:315", "l:346", "l:355", "l:387", "l:396", "l:424", "l:431", "l:460", "l:467", "l:498", "l:505", "l:621", "l:252", "l:285", "l:321", "l:363", "l:403", "l:85", "l:187", "l:186", "l:244", "l:243", "l:277", "l:276", "l:453", "l:523", "l:619", "l:620", "l:524", "l:454", "l:86" ] }, { "active_attention_heads": 22, "active_mlp_neurons": 987, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L13.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:378", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 1, "role": "transport", "source_columns": [ [ 212, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1824, 1 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L13.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:415", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 1, "role": "transport", "source_columns": [ [ 1243, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1825, 1 ] ], "type": "add_into" }, { "heads": [ 2, 1 ], "id": "L13.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:440", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 1, "role": "transport", "source_columns": [ [ 1088, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1826, 1 ] ], "type": "add_into" }, { "heads": [ 3, 1 ], "id": "L13.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": "l:476", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 1, "role": "transport", "source_columns": [ [ 1030, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1827, 1 ] ], "type": "add_into" }, { "heads": [ 4, 1 ], "id": "L13.attn.4", "key_source_columns": [ [ 0, 1 ] ], "node": "l:510", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 1, "role": "transport", "source_columns": [ [ 1086, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1828, 1 ] ], "type": "add_into" }, { "heads": [ 5, 1 ], "id": "L13.attn.5", "key_source_columns": [ [ 0, 1 ] ], "node": "l:543", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 1, "role": "transport", "source_columns": [ [ 1085, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1829, 1 ] ], "type": "add_into" }, { "heads": [ 6, 1 ], "id": "L13.attn.6", "key_source_columns": [ [ 0, 1 ] ], "node": "l:582", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 1, "role": "transport", "source_columns": [ [ 1084, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1830, 1 ] ], "type": "add_into" }, { "heads": [ 7, 1 ], "id": "L13.attn.7", "key_source_columns": [ [ 0, 1 ] ], "node": "l:354", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 174, 1 ] ], "source_columns_b": null, "target_columns": [ [ 52, 1 ] ], "type": "compute_linear" }, { "heads": [ 8, 1 ], "id": "L13.attn.8", "key_source_columns": [ [ 0, 1 ] ], "node": "l:395", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 286, 1 ] ], "source_columns_b": null, "target_columns": [ [ 72, 1 ] ], "type": "compute_linear" }, { "heads": [ 9, 1 ], "id": "L13.attn.9", "key_source_columns": [ [ 0, 1 ] ], "node": "l:430", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 287, 1 ] ], "source_columns_b": null, "target_columns": [ [ 73, 1 ] ], "type": "compute_linear" }, { "heads": [ 10, 1 ], "id": "L13.attn.10", "key_source_columns": [ [ 0, 1 ] ], "node": "l:466", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 288, 1 ] ], "source_columns_b": null, "target_columns": [ [ 74, 1 ] ], "type": "compute_linear" }, { "heads": [ 11, 1 ], "id": "L13.attn.11", "key_source_columns": [ [ 0, 1 ] ], "node": "l:314", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 289, 1 ] ], "source_columns_b": null, "target_columns": [ [ 75, 1 ] ], "type": "compute_linear" }, { "heads": [ 12, 1 ], "id": "L13.attn.12", "key_source_columns": [ [ 0, 1 ] ], "node": "l:222", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1243, 1 ] ], "source_columns_b": [ [ 59, 1 ] ], "target_columns": [ [ 76, 1 ] ], "type": "compute_add" }, { "heads": [ 13, 1 ], "id": "L13.attn.13", "key_source_columns": [ [ 0, 1 ] ], "node": "l:272", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 212, 1 ] ], "source_columns_b": [ [ 59, 1 ] ], "target_columns": [ [ 77, 1 ] ], "type": "compute_add" }, { "heads": [ 14, 1 ], "id": "L13.attn.14", "key_source_columns": [ [ 0, 1 ] ], "node": "l:251", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1084, 1 ] ], "source_columns_b": [ [ 60, 1 ] ], "target_columns": [ [ 78, 1 ] ], "type": "compute_add" }, { "heads": [ 15, 1 ], "id": "L13.attn.15", "key_source_columns": [ [ 0, 1 ] ], "node": "l:284", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1085, 1 ] ], "source_columns_b": [ [ 60, 1 ] ], "target_columns": [ [ 79, 1 ] ], "type": "compute_add" }, { "heads": [ 16, 1 ], "id": "L13.attn.16", "key_source_columns": [ [ 0, 1 ] ], "node": "l:320", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1086, 1 ] ], "source_columns_b": [ [ 60, 1 ] ], "target_columns": [ [ 80, 1 ] ], "type": "compute_add" }, { "heads": [ 17, 1 ], "id": "L13.attn.17", "key_source_columns": [ [ 0, 1 ] ], "node": "l:362", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1030, 1 ] ], "source_columns_b": [ [ 60, 1 ] ], "target_columns": [ [ 81, 1 ] ], "type": "compute_add" }, { "heads": [ 18, 1 ], "id": "L13.attn.18", "key_source_columns": [ [ 0, 1 ] ], "node": "l:402", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1088, 1 ] ], "source_columns_b": [ [ 60, 1 ] ], "target_columns": [ [ 82, 1 ] ], "type": "compute_add" }, { "heads": [ 19, 3 ], "id": "L13.attn.19", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 1 ], [ 36, 1 ], [ 39, 1 ], [ 41, 1 ], [ 43, 3 ], [ 49, 9 ], [ 59, 2 ], [ 67, 1 ], [ 72, 12 ], [ 130, 3 ], [ 143, 9 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 173, 2 ], [ 177, 9 ], [ 192, 1 ], [ 195, 3 ], [ 199, 3 ], [ 222, 6 ], [ 229, 3 ], [ 245, 1 ], [ 254, 30 ], [ 286, 4 ], [ 332, 1 ], [ 339, 4 ], [ 344, 2 ], [ 669, 6 ], [ 868, 1 ], [ 960, 5 ], [ 990, 1 ], [ 1025, 1 ], [ 1150, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 22, "index": 13, "mlp_operations": [ { "hidden_slots": [ [ 1, 134 ] ], "id": "L13.mlp.0", "node": "l:581", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1830, 1 ] ], "source_columns_b": null, "target_columns": [ [ 34, 1 ], [ 36, 1 ], [ 44, 2 ], [ 83, 1 ], [ 143, 9 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 178, 7 ], [ 222, 1 ], [ 229, 1 ], [ 332, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 135, 206 ] ], "id": "L13.mlp.1", "node": "l:542", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1829, 1 ] ], "source_columns_b": null, "target_columns": [ [ 49, 3 ], [ 53, 2 ], [ 173, 1 ], [ 223, 5 ], [ 231, 1 ], [ 245, 2 ], [ 254, 13 ], [ 268, 3 ], [ 273, 1 ], [ 868, 1 ], [ 1150, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 341, 5 ] ], "id": "L13.mlp.2", "node": "l:536", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 346, 1 ] ], "source_columns_b": null, "target_columns": [ [ 55, 3 ], [ 59, 2 ], [ 67, 1 ], [ 130, 3 ], [ 174, 1 ], [ 185, 1 ], [ 197, 1 ], [ 200, 1 ], [ 230, 1 ], [ 274, 3 ], [ 278, 4 ], [ 283, 1 ], [ 286, 1 ], [ 288, 2 ], [ 339, 4 ], [ 344, 2 ], [ 669, 1 ], [ 1025, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 346, 206 ] ], "id": "L13.mlp.3", "node": "l:509", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1828, 1 ] ], "source_columns_b": null, "target_columns": [ [ 39, 1 ], [ 41, 1 ], [ 43, 1 ], [ 177, 1 ], [ 192, 1 ], [ 195, 2 ], [ 199, 1 ], [ 201, 1 ], [ 670, 5 ], [ 959, 6 ], [ 988, 5 ], [ 1031, 1 ], [ 1089, 3 ], [ 1093, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 552, 62 ] ], "id": "L13.mlp.4", "node": "l:439", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1826, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1097, 14 ], [ 1112, 19 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 614, 82 ] ], "id": "L13.mlp.5", "node": "l:414", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1825, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1131, 15 ], [ 1152, 18 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 696, 6 ] ], "id": "L13.mlp.6", "node": "l:401", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 82, 1 ] ], "source_columns_b": null, "target_columns": [ [ 42, 1 ], [ 47, 2 ], [ 172, 1 ], [ 237, 4 ], [ 244, 1 ], [ 335, 1 ], [ 881, 1 ], [ 1019, 1 ], [ 1049, 2 ], [ 1149, 1 ], [ 1170, 9 ], [ 1184, 6 ], [ 1223, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 702, 82 ] ], "id": "L13.mlp.7", "node": "l:377", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1824, 1 ] ], "source_columns_b": null, "target_columns": [ [ 247, 4 ], [ 1060, 1 ], [ 1062, 1 ], [ 1064, 1 ], [ 1226, 12 ], [ 1239, 4 ], [ 1244, 3 ], [ 1249, 4 ], [ 1254, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 784, 14 ] ], "id": "L13.mlp.8", "node": "l:361", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 81, 1 ] ], "source_columns_b": null, "target_columns": [ [ 251, 3 ], [ 267, 1 ], [ 271, 2 ], [ 277, 1 ], [ 282, 1 ], [ 284, 2 ], [ 287, 1 ], [ 292, 1 ], [ 307, 1 ], [ 312, 1 ], [ 317, 1 ], [ 321, 1 ], [ 323, 1 ], [ 439, 1 ], [ 514, 1 ], [ 1027, 1 ], [ 1065, 3 ], [ 1092, 1 ], [ 1111, 1 ], [ 1146, 3 ], [ 1151, 1 ], [ 1238, 1 ], [ 1247, 2 ], [ 1253, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 798, 22 ] ], "id": "L13.mlp.9", "node": "l:319", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 80, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1257, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 820, 22 ] ], "id": "L13.mlp.10", "node": "l:283", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 79, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1290, 24 ], [ 1332, 1 ], [ 1357, 1 ], [ 1369, 1 ], [ 1380, 6 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 842, 10 ] ], "id": "L13.mlp.11", "node": "l:271", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 77, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1386, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 852, 14 ] ], "id": "L13.mlp.12", "node": "l:250", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 78, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1419, 13 ], [ 1433, 20 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 866, 10 ] ], "id": "L13.mlp.13", "node": "l:221", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 76, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1453, 4 ], [ 1458, 29 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 876, 33 ] ], "id": "L13.mlp.14", "node": "l:84", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 38, 1 ], [ 40, 1 ], [ 66, 1 ], [ 133, 7 ], [ 188, 4 ], [ 233, 2 ], [ 331, 1 ], [ 675, 9 ], [ 966, 6 ], [ 1008, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1487, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 909, 33 ] ], "id": "L13.mlp.15", "node": "l:452", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 37, 1 ], [ 69, 1 ], [ 109, 6 ], [ 118, 2 ], [ 186, 2 ], [ 232, 1 ], [ 496, 18 ], [ 515, 1 ], [ 930, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1520, 15 ], [ 1536, 18 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 942, 33 ] ], "id": "L13.mlp.16", "node": "l:618", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 61, 5 ], [ 297, 10 ], [ 308, 4 ], [ 313, 4 ], [ 318, 3 ], [ 322, 1 ], [ 324, 3 ], [ 328, 1 ], [ 333, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1554, 4 ], [ 1559, 29 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 975, 2 ] ], "id": "L13.mlp.17", "node": "l:433", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1822, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1588, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 977, 2 ] ], "id": "L13.mlp.18", "node": "l:504", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 294, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1589, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 979, 2 ] ], "id": "L13.mlp.19", "node": "l:345", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 52, 1 ] ], "source_columns_b": null, "target_columns": [ [ 290, 1 ] ], "type": "add_into_bypass" }, { "hidden_slots": [ [ 981, 2 ] ], "id": "L13.mlp.20", "node": "l:386", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 72, 1 ] ], "source_columns_b": null, "target_columns": [ [ 291, 1 ] ], "type": "add_into_bypass" }, { "hidden_slots": [ [ 983, 2 ] ], "id": "L13.mlp.21", "node": "l:423", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 73, 1 ] ], "source_columns_b": null, "target_columns": [ [ 58, 1 ] ], "type": "add_into_bypass" }, { "hidden_slots": [ [ 985, 2 ] ], "id": "L13.mlp.22", "node": "l:459", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 74, 1 ] ], "source_columns_b": null, "target_columns": [ [ 293, 1 ] ], "type": "add_into_bypass" } ], "newly_computed": [ "l:222", "l:272", "l:378", "l:415", "l:314", "l:354", "l:345", "l:395", "l:386", "l:430", "l:423", "l:433", "l:466", "l:459", "l:504", "l:251", "l:284", "l:320", "l:362", "l:402", "l:440", "l:476", "l:510", "l:543", "l:582", "l:84", "l:83", "l:220", "l:249", "l:270", "l:282", "l:318", "l:360", "l:376", "l:400", "l:413", "l:438", "l:452", "l:508", "l:535", "l:541", "l:580", "l:618", "l:593", "l:581", "l:542", "l:536", "l:509", "l:439", "l:414", "l:401", "l:377", "l:361", "l:319", "l:283", "l:271", "l:250", "l:221" ] }, { "active_attention_heads": 16, "active_mlp_neurons": 1221, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L14.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:307", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 75, 1 ] ], "source_columns_b": null, "target_columns": [ [ 295, 1 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L14.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:344", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 1833, 1 ] ], "source_columns_b": null, "target_columns": [ [ 290, 1 ] ], "type": "add_into" }, { "heads": [ 2, 1 ], "id": "L14.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:385", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 1834, 1 ] ], "source_columns_b": null, "target_columns": [ [ 291, 1 ] ], "type": "add_into" }, { "heads": [ 3, 1 ], "id": "L14.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": "l:422", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 1588, 1 ] ], "source_columns_b": null, "target_columns": [ [ 58, 1 ] ], "type": "add_into" }, { "heads": [ 4, 1 ], "id": "L14.attn.4", "key_source_columns": [ [ 0, 1 ] ], "node": "l:458", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 982, 1 ] ], "source_columns_b": null, "target_columns": [ [ 293, 1 ] ], "type": "add_into" }, { "heads": [ 5, 1 ], "id": "L14.attn.5", "key_source_columns": [ [ 0, 1 ] ], "node": "l:497", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 1589, 1 ] ], "source_columns_b": null, "target_columns": [ [ 296, 1 ] ], "type": "add_into" }, { "heads": [ 6, 2 ], "id": "L14.attn.6", "key_source_columns": [ [ 0, 1 ] ], "node": "l:82", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1487, 33 ], [ 35, 1 ], [ 228, 1 ], [ 327, 1 ], [ 869, 8 ], [ 878, 1 ], [ 883, 5 ], [ 965, 1 ], [ 972, 10 ], [ 984, 2 ], [ 1006, 2 ], [ 1009, 1 ], [ 444, 33 ] ], "source_columns_b": null, "target_columns": [ [ 35, 1 ], [ 37, 2 ], [ 40, 1 ], [ 52, 1 ], [ 61, 6 ], [ 69, 1 ], [ 72, 3 ], [ 76, 2 ], [ 79, 4 ], [ 94, 12 ] ], "type": "compute_linear" }, { "heads": [ 8, 2 ], "id": "L14.attn.7", "key_source_columns": [ [ 0, 1 ] ], "node": "l:592", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 516, 33 ], [ 94, 15 ], [ 115, 3 ], [ 120, 10 ], [ 241, 1 ], [ 330, 1 ], [ 477, 3 ], [ 1554, 4 ], [ 1559, 29 ] ], "source_columns_b": null, "target_columns": [ [ 106, 24 ], [ 133, 7 ], [ 186, 2 ] ], "type": "compute_linear" }, { "heads": [ 10, 6 ], "id": "L14.attn.8", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 35, 1 ], [ 37, 2 ], [ 40, 1 ], [ 52, 1 ], [ 61, 6 ], [ 69, 1 ], [ 72, 3 ], [ 76, 2 ], [ 79, 4 ], [ 94, 36 ], [ 133, 7 ], [ 186, 6 ], [ 212, 1 ], [ 228, 1 ], [ 232, 3 ], [ 241, 1 ], [ 294, 1 ], [ 297, 10 ], [ 308, 4 ], [ 313, 4 ], [ 318, 3 ], [ 322, 1 ], [ 324, 5 ], [ 330, 2 ], [ 333, 2 ], [ 346, 1 ], [ 444, 36 ], [ 496, 18 ], [ 515, 34 ], [ 675, 9 ], [ 869, 8 ], [ 878, 1 ], [ 883, 5 ], [ 930, 1 ], [ 965, 17 ], [ 984, 2 ], [ 1006, 4 ], [ 1030, 1 ], [ 1084, 3 ], [ 1088, 1 ], [ 1243, 1 ], [ 1487, 33 ], [ 1554, 4 ], [ 1559, 29 ], [ 1822, 1 ], [ 1824, 3 ], [ 1828, 3 ] ], "type": "cancel" } ], "emitted_attention_heads": 16, "index": 14, "mlp_operations": [ { "hidden_slots": [ [ 1, 73 ] ], "id": "L14.mlp.0", "node": "l:496", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 296, 1 ] ], "source_columns_b": null, "target_columns": [ [ 188, 4 ], [ 212, 1 ], [ 228, 1 ], [ 232, 3 ], [ 241, 1 ], [ 294, 1 ], [ 297, 10 ], [ 308, 4 ], [ 313, 4 ], [ 318, 3 ], [ 322, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 74, 134 ] ], "id": "L14.mlp.1", "node": "l:475", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1827, 1 ] ], "source_columns_b": null, "target_columns": [ [ 324, 5 ], [ 330, 2 ], [ 333, 2 ], [ 346, 1 ], [ 444, 23 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 208, 81 ] ], "id": "L14.mlp.2", "node": "l:457", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 293, 1 ] ], "source_columns_b": null, "target_columns": [ [ 467, 13 ], [ 496, 18 ], [ 515, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 289, 81 ] ], "id": "L14.mlp.3", "node": "l:421", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 58, 1 ] ], "source_columns_b": null, "target_columns": [ [ 517, 32 ], [ 675, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 370, 9 ] ], "id": "L14.mlp.4", "node": "l:384", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 291, 1 ] ], "source_columns_b": null, "target_columns": [ [ 676, 8 ], [ 869, 8 ], [ 878, 1 ], [ 883, 5 ], [ 930, 1 ], [ 965, 10 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 379, 9 ] ], "id": "L14.mlp.5", "node": "l:343", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 290, 1 ] ], "source_columns_b": null, "target_columns": [ [ 975, 7 ], [ 984, 2 ], [ 1006, 4 ], [ 1030, 1 ], [ 1084, 3 ], [ 1088, 1 ], [ 1243, 1 ], [ 1487, 14 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 388, 4 ] ], "id": "L14.mlp.6", "node": "l:306", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 295, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1501, 19 ], [ 1554, 4 ], [ 1559, 10 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 392, 33 ] ], "id": "L14.mlp.7", "node": "l:219", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 1453, 4 ], [ 1458, 29 ] ], "source_columns_b": null, "target_columns": [ [ 1569, 12 ], [ 1590, 21 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 425, 33 ] ], "id": "L14.mlp.8", "node": "l:248", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 1419, 13 ], [ 1433, 20 ] ], "source_columns_b": null, "target_columns": [ [ 1611, 27 ], [ 1639, 6 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 458, 33 ] ], "id": "L14.mlp.9", "node": "l:269", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 1386, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1645, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 491, 33 ] ], "id": "L14.mlp.10", "node": "l:281", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 1290, 24 ], [ 1332, 1 ], [ 1357, 1 ], [ 1369, 1 ], [ 1380, 6 ] ], "source_columns_b": null, "target_columns": [ [ 1680, 1 ], [ 1683, 21 ], [ 1705, 11 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 524, 33 ] ], "id": "L14.mlp.11", "node": "l:359", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 251, 3 ], [ 267, 1 ], [ 271, 2 ], [ 277, 1 ], [ 282, 1 ], [ 284, 2 ], [ 287, 1 ], [ 292, 1 ], [ 307, 1 ], [ 312, 1 ], [ 317, 1 ], [ 321, 1 ], [ 323, 1 ], [ 439, 1 ], [ 514, 1 ], [ 1027, 1 ], [ 1065, 3 ], [ 1092, 1 ], [ 1111, 1 ], [ 1146, 3 ], [ 1151, 1 ], [ 1238, 1 ], [ 1247, 2 ], [ 1253, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1716, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 557, 33 ] ], "id": "L14.mlp.12", "node": "l:399", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 42, 1 ], [ 47, 2 ], [ 172, 1 ], [ 237, 4 ], [ 244, 1 ], [ 335, 1 ], [ 881, 1 ], [ 1019, 1 ], [ 1049, 2 ], [ 1149, 1 ], [ 1170, 9 ], [ 1184, 6 ], [ 1223, 3 ] ], "source_columns_b": null, "target_columns": [ [ 1749, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 590, 33 ] ], "id": "L14.mlp.13", "node": "l:412", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 1131, 15 ], [ 1152, 18 ] ], "source_columns_b": null, "target_columns": [ [ 1782, 8 ], [ 1791, 25 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 623, 33 ] ], "id": "L14.mlp.14", "node": "l:507", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 39, 1 ], [ 41, 1 ], [ 43, 1 ], [ 177, 1 ], [ 192, 1 ], [ 195, 2 ], [ 199, 1 ], [ 201, 1 ], [ 670, 5 ], [ 959, 6 ], [ 988, 5 ], [ 1031, 1 ], [ 1089, 3 ], [ 1093, 4 ] ], "source_columns_b": null, "target_columns": [ [ 1816, 11 ], [ 1828, 3 ], [ 1832, 1 ], [ 1835, 6 ], [ 1842, 12 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 656, 33 ] ], "id": "L14.mlp.15", "node": "l:522", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 46, 1 ], [ 68, 1 ], [ 70, 2 ], [ 84, 8 ], [ 329, 1 ], [ 336, 3 ], [ 480, 16 ], [ 882, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1854, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 689, 33 ] ], "id": "L14.mlp.16", "node": "l:534", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 55, 3 ], [ 59, 2 ], [ 67, 1 ], [ 130, 3 ], [ 174, 1 ], [ 185, 1 ], [ 197, 1 ], [ 200, 1 ], [ 230, 1 ], [ 274, 3 ], [ 278, 4 ], [ 283, 1 ], [ 286, 1 ], [ 288, 2 ], [ 339, 4 ], [ 344, 2 ], [ 669, 1 ], [ 1025, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1887, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 722, 33 ] ], "id": "L14.mlp.17", "node": "l:540", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 49, 3 ], [ 53, 2 ], [ 173, 1 ], [ 223, 5 ], [ 231, 1 ], [ 245, 2 ], [ 254, 13 ], [ 268, 3 ], [ 273, 1 ], [ 868, 1 ], [ 1150, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1920, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 755, 33 ] ], "id": "L14.mlp.18", "node": "l:579", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 34, 1 ], [ 36, 1 ], [ 44, 2 ], [ 83, 1 ], [ 143, 9 ], [ 158, 7 ], [ 168, 1 ], [ 171, 1 ], [ 178, 7 ], [ 222, 1 ], [ 229, 1 ], [ 332, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1953, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 788, 33 ] ], "id": "L14.mlp.19", "node": "l:76", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1023, 1 ], [ 35, 1 ], [ 37, 2 ], [ 40, 1 ], [ 52, 1 ], [ 61, 6 ], [ 69, 1 ], [ 72, 3 ], [ 76, 2 ], [ 79, 4 ], [ 94, 12 ] ], "source_columns_b": null, "target_columns": [ [ 1581, 6 ], [ 1986, 26 ], [ 2045, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 821, 66 ] ], "id": "L14.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 49, 3 ], [ 53, 2 ], [ 173, 1 ], [ 223, 5 ], [ 231, 1 ], [ 245, 2 ], [ 254, 13 ], [ 268, 3 ], [ 273, 1 ], [ 868, 1 ], [ 1150, 1 ] ], "source_columns_b": null, "target_columns": [ [ 49, 3 ], [ 53, 2 ], [ 173, 1 ], [ 223, 5 ], [ 231, 1 ], [ 245, 2 ], [ 254, 13 ], [ 268, 3 ], [ 273, 1 ], [ 868, 1 ], [ 1150, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 887, 66 ] ], "id": "L14.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1131, 15 ], [ 1152, 18 ] ], "source_columns_b": null, "target_columns": [ [ 1131, 15 ], [ 1152, 18 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 953, 66 ] ], "id": "L14.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 251, 3 ], [ 267, 1 ], [ 271, 2 ], [ 277, 1 ], [ 282, 1 ], [ 284, 2 ], [ 287, 1 ], [ 292, 1 ], [ 307, 1 ], [ 312, 1 ], [ 317, 1 ], [ 321, 1 ], [ 323, 1 ], [ 439, 1 ], [ 514, 1 ], [ 1027, 1 ], [ 1065, 3 ], [ 1092, 1 ], [ 1111, 1 ], [ 1146, 3 ], [ 1151, 1 ], [ 1238, 1 ], [ 1247, 2 ], [ 1253, 1 ] ], "source_columns_b": null, "target_columns": [ [ 251, 3 ], [ 267, 1 ], [ 271, 2 ], [ 277, 1 ], [ 282, 1 ], [ 284, 2 ], [ 287, 1 ], [ 292, 1 ], [ 307, 1 ], [ 312, 1 ], [ 317, 1 ], [ 321, 1 ], [ 323, 1 ], [ 439, 1 ], [ 514, 1 ], [ 1027, 1 ], [ 1065, 3 ], [ 1092, 1 ], [ 1111, 1 ], [ 1146, 3 ], [ 1151, 1 ], [ 1238, 1 ], [ 1247, 2 ], [ 1253, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1019, 66 ] ], "id": "L14.mlp.23", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1386, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1386, 33 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1085, 66 ] ], "id": "L14.mlp.24", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1419, 13 ], [ 1433, 20 ] ], "source_columns_b": null, "target_columns": [ [ 1419, 13 ], [ 1433, 20 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1151, 66 ] ], "id": "L14.mlp.25", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1453, 4 ], [ 1458, 29 ] ], "source_columns_b": null, "target_columns": [ [ 1453, 4 ], [ 1458, 29 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1217, 2 ] ], "id": "L14.mlp.26", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 78, 1 ] ], "source_columns_b": null, "target_columns": [ [ 78, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1219, 2 ] ], "id": "L14.mlp.27", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1023, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1023, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:307", "l:344", "l:385", "l:422", "l:458", "l:497", "l:82", "l:219", "l:248", "l:218", "l:269", "l:281", "l:268", "l:305", "l:342", "l:359", "l:383", "l:399", "l:412", "l:420", "l:456", "l:474", "l:495", "l:507", "l:522", "l:534", "l:540", "l:521", "l:579", "l:554", "l:592", "l:77", "l:76", "l:587", "l:496", "l:475", "l:457", "l:421", "l:384", "l:343", "l:306" ] }, { "active_attention_heads": 11, "active_mlp_neurons": 904, "attention_operations": [ { "heads": [ 0, 2 ], "id": "L15.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:217", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1569, 12 ], [ 1590, 21 ], [ 198, 1 ], [ 203, 1 ], [ 983, 1 ], [ 986, 2 ], [ 993, 6 ], [ 1000, 6 ], [ 1010, 9 ], [ 1032, 6 ], [ 1042, 1 ], [ 1611, 27 ], [ 1639, 6 ] ], "source_columns_b": null, "target_columns": [ [ 34, 14 ], [ 49, 18 ], [ 69, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 2 ], "id": "L15.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:520", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1854, 99 ] ], "source_columns_b": null, "target_columns": [ [ 67, 2 ], [ 70, 13 ], [ 84, 7 ], [ 96, 3 ], [ 130, 3 ], [ 143, 3 ], [ 1643, 1 ], [ 1857, 1 ] ], "type": "compute_linear" }, { "heads": [ 4, 7 ], "id": "L15.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 15 ], [ 52, 1 ], [ 55, 23 ], [ 79, 13 ], [ 94, 12 ], [ 130, 3 ], [ 143, 9 ], [ 158, 7 ], [ 168, 1 ], [ 171, 2 ], [ 174, 1 ], [ 177, 9 ], [ 192, 1 ], [ 195, 7 ], [ 203, 1 ], [ 222, 1 ], [ 229, 2 ], [ 237, 4 ], [ 244, 1 ], [ 274, 3 ], [ 278, 4 ], [ 283, 1 ], [ 286, 1 ], [ 288, 4 ], [ 293, 1 ], [ 296, 1 ], [ 329, 1 ], [ 332, 1 ], [ 335, 8 ], [ 344, 2 ], [ 480, 16 ], [ 669, 6 ], [ 881, 2 ], [ 959, 6 ], [ 982, 2 ], [ 986, 13 ], [ 1000, 6 ], [ 1010, 10 ], [ 1025, 1 ], [ 1031, 7 ], [ 1042, 1 ], [ 1049, 2 ], [ 1089, 3 ], [ 1093, 4 ], [ 1149, 1 ], [ 1170, 9 ], [ 1184, 6 ], [ 1223, 3 ], [ 1290, 24 ], [ 1332, 1 ], [ 1357, 1 ], [ 1369, 1 ], [ 1380, 6 ], [ 1569, 12 ], [ 1588, 50 ], [ 1639, 6 ], [ 1827, 1 ], [ 1833, 2 ], [ 1854, 99 ] ], "type": "cancel" } ], "emitted_attention_heads": 11, "index": 15, "mlp_operations": [ { "hidden_slots": [ [ 1, 33 ] ], "id": "L15.mlp.0", "node": "l:304", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 1501, 19 ], [ 1554, 4 ], [ 1559, 10 ] ], "source_columns_b": null, "target_columns": [ [ 146, 6 ], [ 158, 7 ], [ 168, 1 ], [ 171, 4 ], [ 177, 9 ], [ 192, 1 ], [ 195, 5 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 34, 33 ] ], "id": "L15.mlp.1", "node": "l:317", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 1257, 33 ] ], "source_columns_b": null, "target_columns": [ [ 48, 1 ], [ 200, 2 ], [ 203, 1 ], [ 222, 6 ], [ 229, 3 ], [ 237, 4 ], [ 244, 3 ], [ 254, 13 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 67, 33 ] ], "id": "L15.mlp.2", "node": "l:341", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 975, 7 ], [ 984, 2 ], [ 1006, 4 ], [ 1030, 1 ], [ 1084, 3 ], [ 1088, 1 ], [ 1243, 1 ], [ 1487, 14 ] ], "source_columns_b": null, "target_columns": [ [ 267, 27 ], [ 296, 1 ], [ 321, 1 ], [ 323, 1 ], [ 329, 1 ], [ 332, 1 ], [ 1858, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 100, 33 ] ], "id": "L15.mlp.3", "node": "l:375", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1678, 1 ], [ 247, 4 ], [ 1060, 1 ], [ 1062, 1 ], [ 1064, 1 ], [ 1226, 12 ], [ 1239, 4 ], [ 1244, 3 ], [ 1249, 4 ], [ 1254, 3 ] ], "source_columns_b": null, "target_columns": [ [ 83, 1 ], [ 91, 1 ], [ 94, 2 ], [ 99, 2 ], [ 335, 8 ], [ 344, 2 ], [ 439, 1 ], [ 480, 16 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 133, 33 ] ], "id": "L15.mlp.4", "node": "l:382", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 676, 8 ], [ 869, 8 ], [ 878, 1 ], [ 883, 5 ], [ 930, 1 ], [ 965, 10 ] ], "source_columns_b": null, "target_columns": [ [ 101, 5 ], [ 514, 1 ], [ 669, 6 ], [ 868, 1 ], [ 881, 2 ], [ 959, 6 ], [ 982, 2 ], [ 986, 10 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 166, 33 ] ], "id": "L15.mlp.5", "node": "l:419", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 517, 32 ], [ 675, 1 ] ], "source_columns_b": null, "target_columns": [ [ 996, 3 ], [ 1000, 6 ], [ 1010, 10 ], [ 1023, 1 ], [ 1025, 1 ], [ 1027, 1 ], [ 1031, 7 ], [ 1042, 1 ], [ 1049, 2 ], [ 1066, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 199, 33 ] ], "id": "L15.mlp.6", "node": "l:437", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 1097, 14 ], [ 1112, 19 ] ], "source_columns_b": null, "target_columns": [ [ 1065, 1 ], [ 1067, 1 ], [ 1089, 8 ], [ 1111, 1 ], [ 1131, 15 ], [ 1147, 2 ], [ 1150, 1 ], [ 1152, 3 ], [ 1855, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 232, 33 ] ], "id": "L15.mlp.7", "node": "l:455", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 467, 13 ], [ 496, 18 ], [ 515, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1155, 24 ], [ 1184, 6 ], [ 1223, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 265, 33 ] ], "id": "L15.mlp.8", "node": "l:473", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1679, 1 ], [ 324, 5 ], [ 330, 2 ], [ 333, 2 ], [ 346, 1 ], [ 444, 23 ] ], "source_columns_b": null, "target_columns": [ [ 251, 3 ], [ 1238, 1 ], [ 1247, 2 ], [ 1253, 1 ], [ 1290, 12 ], [ 1303, 1 ], [ 1305, 1 ], [ 1876, 1 ], [ 1878, 11 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 298, 33 ] ], "id": "L15.mlp.9", "node": "l:494", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1638, 1 ], [ 188, 4 ], [ 212, 1 ], [ 228, 1 ], [ 232, 3 ], [ 241, 1 ], [ 294, 1 ], [ 297, 10 ], [ 308, 4 ], [ 313, 4 ], [ 318, 3 ], [ 322, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1306, 8 ], [ 1332, 1 ], [ 1357, 1 ], [ 1369, 1 ], [ 1380, 19 ], [ 1903, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 331, 2 ] ], "id": "L15.mlp.10", "node": "l:263", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1020, 1 ], [ 1831, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1399, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 333, 33 ] ], "id": "L15.mlp.11", "node": "l:211", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1041, 1 ], [ 34, 14 ], [ 49, 18 ], [ 69, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1400, 32 ], [ 1913, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 366, 33 ] ], "id": "L15.mlp.12", "node": "l:514", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1535, 1 ], [ 67, 2 ], [ 70, 13 ], [ 84, 7 ], [ 96, 3 ], [ 130, 3 ], [ 143, 3 ], [ 1643, 1 ], [ 1857, 1 ] ], "source_columns_b": null, "target_columns": [ [ 307, 1 ], [ 312, 1 ], [ 317, 1 ], [ 1146, 1 ], [ 1149, 1 ], [ 1151, 1 ], [ 1302, 1 ], [ 1304, 1 ], [ 1433, 24 ], [ 1458, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 399, 33 ] ], "id": "L15.mlp.13", "node": "l:586", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1457, 1 ], [ 106, 24 ], [ 133, 7 ], [ 186, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1459, 28 ], [ 1569, 5 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 432, 66 ] ], "id": "L15.mlp.14", "node": "l:267", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1645, 33 ], [ 1026, 1 ], [ 1038, 3 ], [ 1043, 3 ], [ 1047, 2 ], [ 1052, 8 ], [ 1068, 13 ], [ 1082, 2 ], [ 1087, 1 ], [ 1680, 1 ], [ 1683, 21 ], [ 1705, 11 ] ], "source_columns_b": null, "target_columns": [ [ 1574, 7 ], [ 1587, 26 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 498, 66 ] ], "id": "L15.mlp.15", "node": "l:553", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 353, 66 ], [ 1953, 33 ] ], "source_columns_b": null, "target_columns": [ [ 1613, 25 ], [ 1639, 4 ], [ 1644, 1 ], [ 1827, 1 ], [ 1833, 2 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L15.mlp.16", "node": "l:1057", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1854, 1 ], [ 1856, 1 ], [ 1859, 17 ], [ 1877, 1 ], [ 1889, 13 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 564, 66 ] ], "id": "L15.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 188, 4 ], [ 212, 1 ], [ 228, 1 ], [ 232, 3 ], [ 241, 1 ], [ 294, 1 ], [ 297, 10 ], [ 308, 4 ], [ 313, 4 ], [ 318, 3 ], [ 322, 1 ] ], "source_columns_b": null, "target_columns": [ [ 188, 4 ], [ 212, 1 ], [ 228, 1 ], [ 232, 3 ], [ 241, 1 ], [ 294, 1 ], [ 297, 10 ], [ 308, 4 ], [ 313, 4 ], [ 318, 3 ], [ 322, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 630, 66 ] ], "id": "L15.mlp.18", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 676, 8 ], [ 869, 8 ], [ 878, 1 ], [ 883, 5 ], [ 930, 1 ], [ 965, 10 ] ], "source_columns_b": null, "target_columns": [ [ 676, 8 ], [ 869, 8 ], [ 878, 1 ], [ 883, 5 ], [ 930, 1 ], [ 965, 10 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 696, 66 ] ], "id": "L15.mlp.19", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 247, 4 ], [ 1060, 1 ], [ 1062, 1 ], [ 1064, 1 ], [ 1226, 12 ], [ 1239, 4 ], [ 1244, 3 ], [ 1249, 4 ], [ 1254, 3 ] ], "source_columns_b": null, "target_columns": [ [ 247, 4 ], [ 1060, 1 ], [ 1062, 1 ], [ 1064, 1 ], [ 1226, 12 ], [ 1239, 4 ], [ 1244, 3 ], [ 1249, 4 ], [ 1254, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 762, 66 ] ], "id": "L15.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 975, 7 ], [ 984, 2 ], [ 1006, 4 ], [ 1030, 1 ], [ 1084, 3 ], [ 1088, 1 ], [ 1243, 1 ], [ 1487, 14 ] ], "source_columns_b": null, "target_columns": [ [ 975, 7 ], [ 984, 2 ], [ 1006, 4 ], [ 1030, 1 ], [ 1084, 3 ], [ 1088, 1 ], [ 1243, 1 ], [ 1487, 14 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 828, 66 ] ], "id": "L15.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1501, 19 ], [ 1554, 4 ], [ 1559, 10 ] ], "source_columns_b": null, "target_columns": [ [ 1501, 19 ], [ 1554, 4 ], [ 1559, 10 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 894, 2 ] ], "id": "L15.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1638, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1638, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 896, 2 ] ], "id": "L15.mlp.23", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1679, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1679, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 898, 2 ] ], "id": "L15.mlp.24", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 295, 1 ] ], "source_columns_b": null, "target_columns": [ [ 295, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 900, 2 ] ], "id": "L15.mlp.25", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1535, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1535, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 902, 2 ] ], "id": "L15.mlp.26", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1457, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1457, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:217", "l:267", "l:304", "l:317", "l:300", "l:341", "l:334", "l:375", "l:382", "l:374", "l:419", "l:437", "l:411", "l:455", "l:473", "l:451", "l:494", "l:487", "l:520", "l:553", "l:212", "l:211", "l:263", "l:262", "l:515", "l:514", "l:548", "l:586", "l:1057" ] }, { "active_attention_heads": 29, "active_mlp_neurons": 337, "attention_operations": [ { "heads": [ 0, 2 ], "id": "L16.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:299", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 92, 2 ], [ 343, 1 ], [ 347, 6 ], [ 419, 20 ], [ 440, 4 ], [ 146, 6 ], [ 158, 7 ], [ 168, 1 ], [ 171, 4 ], [ 177, 9 ], [ 192, 1 ], [ 195, 5 ], [ 48, 1 ], [ 200, 2 ], [ 203, 1 ], [ 222, 6 ], [ 229, 3 ], [ 237, 4 ], [ 244, 3 ], [ 254, 13 ] ], "source_columns_b": null, "target_columns": [ [ 34, 33 ] ], "type": "compute_linear" }, { "heads": [ 2, 2 ], "id": "L16.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:333", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1314, 18 ], [ 1333, 14 ], [ 1681, 1 ], [ 267, 27 ], [ 296, 1 ], [ 321, 1 ], [ 323, 1 ], [ 329, 1 ], [ 332, 1 ], [ 1858, 1 ], [ 1716, 33 ] ], "source_columns_b": null, "target_columns": [ [ 67, 33 ] ], "type": "compute_linear" }, { "heads": [ 4, 2 ], "id": "L16.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:373", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 83, 1 ], [ 91, 1 ], [ 94, 2 ], [ 99, 2 ], [ 335, 8 ], [ 344, 2 ], [ 439, 1 ], [ 480, 16 ], [ 101, 5 ], [ 514, 1 ], [ 669, 6 ], [ 868, 1 ], [ 881, 2 ], [ 959, 6 ], [ 982, 2 ], [ 986, 10 ], [ 1749, 33 ] ], "source_columns_b": null, "target_columns": [ [ 100, 33 ] ], "type": "compute_linear" }, { "heads": [ 6, 2 ], "id": "L16.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": "l:410", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1782, 8 ], [ 1791, 25 ], [ 996, 3 ], [ 1000, 6 ], [ 1010, 10 ], [ 1023, 1 ], [ 1025, 1 ], [ 1027, 1 ], [ 1031, 7 ], [ 1042, 1 ], [ 1049, 2 ], [ 1066, 1 ], [ 1065, 1 ], [ 1067, 1 ], [ 1089, 8 ], [ 1111, 1 ], [ 1131, 15 ], [ 1147, 2 ], [ 1150, 1 ], [ 1152, 3 ], [ 1855, 1 ] ], "source_columns_b": null, "target_columns": [ [ 133, 7 ], [ 143, 9 ], [ 158, 7 ], [ 168, 1 ], [ 171, 4 ], [ 177, 5 ] ], "type": "compute_linear" }, { "heads": [ 8, 2 ], "id": "L16.attn.4", "key_source_columns": [ [ 0, 1 ] ], "node": "l:450", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1520, 15 ], [ 1536, 18 ], [ 1155, 24 ], [ 1184, 6 ], [ 1223, 3 ], [ 251, 3 ], [ 1238, 1 ], [ 1247, 2 ], [ 1253, 1 ], [ 1290, 12 ], [ 1303, 1 ], [ 1305, 1 ], [ 1876, 1 ], [ 1878, 11 ] ], "source_columns_b": null, "target_columns": [ [ 182, 11 ], [ 195, 7 ], [ 203, 1 ], [ 212, 1 ], [ 222, 13 ] ], "type": "compute_linear" }, { "heads": [ 10, 2 ], "id": "L16.attn.5", "key_source_columns": [ [ 0, 1 ] ], "node": "l:486", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1347, 10 ], [ 1358, 11 ], [ 1370, 10 ], [ 1682, 1 ], [ 1704, 1 ], [ 1306, 8 ], [ 1332, 1 ], [ 1357, 1 ], [ 1369, 1 ], [ 1380, 19 ], [ 1903, 3 ], [ 1816, 11 ], [ 1828, 3 ], [ 1832, 1 ], [ 1835, 6 ], [ 1842, 12 ] ], "source_columns_b": null, "target_columns": [ [ 237, 5 ], [ 244, 28 ] ], "type": "compute_linear" }, { "heads": [ 12, 17 ], "id": "L16.attn.6", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 106 ], [ 143, 9 ], [ 158, 7 ], [ 168, 1 ], [ 171, 4 ], [ 177, 11 ], [ 192, 1 ], [ 195, 7 ], [ 203, 1 ], [ 222, 6 ], [ 229, 3 ], [ 237, 4 ], [ 244, 3 ], [ 251, 43 ], [ 296, 1 ], [ 321, 1 ], [ 323, 226 ], [ 669, 7 ], [ 868, 1 ], [ 881, 2 ], [ 959, 6 ], [ 982, 2 ], [ 986, 13 ], [ 1000, 6 ], [ 1010, 11 ], [ 1023, 1 ], [ 1025, 3 ], [ 1031, 15 ], [ 1047, 4 ], [ 1052, 8 ], [ 1065, 16 ], [ 1082, 2 ], [ 1087, 1 ], [ 1089, 57 ], [ 1147, 2 ], [ 1150, 1 ], [ 1152, 27 ], [ 1184, 6 ], [ 1223, 3 ], [ 1238, 1 ], [ 1247, 2 ], [ 1253, 1 ], [ 1257, 45 ], [ 1303, 1 ], [ 1305, 94 ], [ 1520, 15 ], [ 1536, 18 ], [ 1643, 1 ], [ 1645, 34 ], [ 1680, 110 ], [ 1791, 36 ], [ 1828, 5 ], [ 1835, 6 ], [ 1842, 12 ], [ 1855, 1 ], [ 1857, 2 ], [ 1876, 1 ], [ 1878, 11 ], [ 1903, 3 ], [ 1953, 33 ] ], "type": "cancel" } ], "emitted_attention_heads": 29, "index": 16, "mlp_operations": [ { "hidden_slots": [ [ 1, 33 ] ], "id": "L16.mlp.0", "node": "l:261", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1399, 1 ], [ 1574, 7 ], [ 1587, 26 ] ], "source_columns_b": null, "target_columns": [ [ 272, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 34, 33 ] ], "id": "L16.mlp.1", "node": "l:293", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1046, 1 ], [ 34, 33 ] ], "source_columns_b": null, "target_columns": [ [ 305, 2 ], [ 308, 4 ], [ 313, 4 ], [ 318, 23 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 67, 33 ] ], "id": "L16.mlp.2", "node": "l:327", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1051, 1 ], [ 67, 33 ] ], "source_columns_b": null, "target_columns": [ [ 341, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 100, 33 ] ], "id": "L16.mlp.3", "node": "l:367", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1061, 1 ], [ 100, 33 ] ], "source_columns_b": null, "target_columns": [ [ 374, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 133, 33 ] ], "id": "L16.mlp.4", "node": "l:404", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1063, 1 ], [ 133, 7 ], [ 143, 9 ], [ 158, 7 ], [ 168, 1 ], [ 171, 4 ], [ 177, 5 ] ], "source_columns_b": null, "target_columns": [ [ 407, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 166, 33 ] ], "id": "L16.mlp.5", "node": "l:444", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1558, 1 ], [ 182, 11 ], [ 195, 7 ], [ 203, 1 ], [ 212, 1 ], [ 222, 13 ] ], "source_columns_b": null, "target_columns": [ [ 440, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 199, 33 ] ], "id": "L16.mlp.6", "node": "l:480", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1432, 1 ], [ 237, 5 ], [ 244, 28 ] ], "source_columns_b": null, "target_columns": [ [ 473, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 232, 33 ] ], "id": "L16.mlp.7", "node": "l:547", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1841, 1 ], [ 1613, 25 ], [ 1639, 4 ], [ 1644, 1 ], [ 1827, 1 ], [ 1833, 2 ] ], "source_columns_b": null, "target_columns": [ [ 506, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 265, 66 ] ], "id": "L16.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1613, 25 ], [ 1639, 4 ], [ 1644, 1 ], [ 1827, 1 ], [ 1833, 2 ] ], "source_columns_b": null, "target_columns": [ [ 1613, 25 ], [ 1639, 4 ], [ 1644, 1 ], [ 1827, 1 ], [ 1833, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 331, 2 ] ], "id": "L16.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1046, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1046, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 333, 2 ] ], "id": "L16.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1061, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1061, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 335, 2 ] ], "id": "L16.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1432, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1432, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:299", "l:333", "l:373", "l:410", "l:450", "l:486", "l:261", "l:294", "l:293", "l:328", "l:327", "l:368", "l:367", "l:405", "l:404", "l:445", "l:444", "l:481", "l:480", "l:547", "l:11" ] }, { "active_attention_heads": 32, "active_mlp_neurons": 135, "attention_operations": [ { "heads": [ 0, 14 ], "id": "L17.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:10", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 717, 33 ], [ 1581, 6 ], [ 1986, 26 ], [ 2045, 1 ], [ 1400, 32 ], [ 1913, 1 ], [ 272, 35 ], [ 308, 4 ], [ 313, 4 ], [ 318, 188 ], [ 307, 1 ], [ 312, 1 ], [ 317, 1 ], [ 1146, 1 ], [ 1149, 1 ], [ 1151, 1 ], [ 1302, 1 ], [ 1304, 1 ], [ 1433, 24 ], [ 1458, 1 ], [ 506, 33 ], [ 1459, 28 ], [ 1569, 5 ], [ 750, 12 ], [ 795, 21 ], [ 549, 120 ], [ 702, 12 ], [ 2012, 33 ], [ 816, 12 ], [ 834, 2 ], [ 840, 1 ], [ 845, 2 ], [ 850, 1 ], [ 852, 1 ], [ 855, 1 ], [ 860, 2 ], [ 877, 1 ], [ 879, 2 ], [ 888, 8 ], [ 202, 1 ], [ 204, 8 ], [ 213, 9 ], [ 235, 2 ], [ 1021, 2 ], [ 1024, 1 ], [ 1028, 2 ], [ 1081, 1 ], [ 1179, 5 ], [ 1190, 2 ], [ 714, 3 ], [ 762, 30 ], [ 140, 3 ], [ 152, 6 ], [ 165, 3 ], [ 169, 1 ], [ 193, 2 ], [ 684, 18 ], [ 792, 3 ], [ 828, 6 ], [ 836, 4 ], [ 841, 4 ], [ 847, 3 ], [ 851, 1 ], [ 853, 2 ], [ 856, 4 ], [ 862, 6 ], [ 242, 2 ], [ 1192, 31 ], [ 170, 1 ], [ 175, 2 ], [ 929, 1 ], [ 931, 28 ], [ 999, 1 ], [ 896, 33 ] ], "source_columns_b": null, "target_columns": [ [ 38, 1 ], [ 40, 2 ], [ 43, 1 ], [ 48, 1 ], [ 61, 3 ], [ 107, 18 ], [ 128, 1 ], [ 130, 3 ], [ 327, 1 ], [ 330, 1 ], [ 391, 1 ] ], "type": "compute_linear" }, { "heads": [ 14, 18 ], "id": "L17.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 505 ], [ 549, 120 ], [ 684, 184 ], [ 877, 1 ], [ 879, 2 ], [ 888, 42 ], [ 931, 28 ], [ 999, 1 ], [ 1021, 2 ], [ 1024, 1 ], [ 1028, 2 ], [ 1051, 1 ], [ 1063, 1 ], [ 1081, 1 ], [ 1146, 1 ], [ 1149, 1 ], [ 1151, 1 ], [ 1179, 5 ], [ 1190, 33 ], [ 1302, 1 ], [ 1304, 1 ], [ 1399, 33 ], [ 1433, 24 ], [ 1458, 29 ], [ 1558, 1 ], [ 1569, 44 ], [ 1841, 1 ], [ 1913, 1 ], [ 1986, 60 ] ], "type": "cancel" } ], "emitted_attention_heads": 32, "index": 17, "mlp_operations": [ { "hidden_slots": [ [ 1, 66 ] ], "id": "L17.mlp.0", "node": "l:0", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1790, 1 ], [ 38, 1 ], [ 40, 2 ], [ 43, 1 ], [ 48, 1 ], [ 61, 3 ], [ 107, 18 ], [ 128, 1 ], [ 130, 3 ], [ 327, 1 ], [ 330, 1 ], [ 391, 1 ], [ 1854, 1 ], [ 1856, 1 ], [ 1859, 17 ], [ 1877, 1 ], [ 1889, 13 ] ], "source_columns_b": null, "target_columns": [ [ 1, 33 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 67, 66 ] ], "id": "L17.mlp.1", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1854, 1 ], [ 1856, 1 ], [ 1859, 17 ], [ 1877, 1 ], [ 1889, 13 ] ], "source_columns_b": null, "target_columns": [ [ 1854, 1 ], [ 1856, 1 ], [ 1859, 17 ], [ 1877, 1 ], [ 1889, 13 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 133, 2 ] ], "id": "L17.mlp.2", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1790, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1790, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [], "id": "L17.mlp.3", "node": "i:0", "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 0, 1 ] ], "type": "clear_literal_seed" } ], "newly_computed": [ "l:10", "l:1", "l:0" ] } ], "realizations": [ { "candidates": [ "mlp_composite" ], "node": "l:26", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:28", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:23", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:24", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:27", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:21", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:20", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:19", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:18", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:17", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:44", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:46", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:48", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:43", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:45", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:47", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:41", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:7", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:52", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:53", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:55", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:50", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:49", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:39", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:36", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:37", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:56", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:34", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:62", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:63", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:65", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:60", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:71", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:72", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:74", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:69", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:112", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:125", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:126", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:128", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:123", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:121", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:122", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:119", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:173", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:171", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:109", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:110", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:113", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:107", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:114", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:106", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:105", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:104", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:103", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:132", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:133", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:130", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:129", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:117", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:134", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:137", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:135", "selected": "mlp_composite" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:115", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:140", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:138", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:162", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:163", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:165", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:160", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:166", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:159", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:158", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:157", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:156", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:177", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:178", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:175", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:174", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:169", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:179", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:182", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:180", "selected": "mlp_composite" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:167", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:185", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:183", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:99", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:100", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:102", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:97", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:152", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:153", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:155", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:150", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:198", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:194", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:195", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:197", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:192", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:231", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:227", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:228", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:230", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:225", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:95", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:91", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:92", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:94", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:89", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:206", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:202", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:203", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:205", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:200", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:240", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:236", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:237", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:239", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:234", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:148", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:144", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:145", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:147", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:142", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:274", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:224", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:88", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:275", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:233", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:141", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:87", "selected": "attn_add" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:223", "selected": "attn_add" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:273", "selected": "attn_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:242", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:222", "selected": "attn_add" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:272", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:340", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:339", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:338", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:381", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:380", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:379", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:378", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:418", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:417", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:416", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:415", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:493", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:492", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:491", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:526", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:533", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:532", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:531", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:530", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:528", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:527", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:525", "selected": "attn_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:559", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:566", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:565", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:564", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:563", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:561", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:560", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:558", "selected": "mlp_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:598", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:605", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:604", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:603", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:602", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:600", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:599", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:597", "selected": "mlp_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:664", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:665", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:662", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:661", "selected": "attn_transport" }, { "candidates": [ "mlp_literal" ], "node": "l:655", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:660", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:659", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:658", "selected": "mlp_bypass" }, { "candidates": [ "attn_heads" ], "node": "l:656", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:652", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:717", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:716", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:715", "selected": "mlp_bypass" }, { "candidates": [ "attn_heads" ], "node": "l:718", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:712", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:766", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:765", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:764", "selected": "mlp_bypass" }, { "candidates": [ "attn_heads" ], "node": "l:767", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:761", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:814", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:813", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:812", "selected": "mlp_bypass" }, { "candidates": [ "attn_heads" ], "node": "l:815", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:809", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:57", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:632", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:857", "selected": "mlp_literal" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:393", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:352", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:312", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:394", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:353", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:313", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:191", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:189", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:188", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:246", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:245", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:279", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:278", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:358", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:311", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:310", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:308", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:315", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:314", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:307", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:351", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:350", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:349", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:348", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:346", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:355", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:354", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:345", "selected": "mlp_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:357", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:344", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:392", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:391", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:390", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:389", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:387", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:396", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:395", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:386", "selected": "mlp_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:398", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:385", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:429", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:428", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:427", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:426", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:424", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:431", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:436", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:435", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:434", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:430", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:423", "selected": "mlp_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:433", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:422", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:465", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:464", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:463", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:462", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:460", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:467", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:472", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:471", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:470", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:466", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:459", "selected": "mlp_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:469", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:458", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:503", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:502", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:501", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:500", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:498", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:505", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:504", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:497", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:539", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:538", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:537", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:571", "selected": "attn_transport" }, { "candidates": [ "mlp_literal" ], "node": "l:578", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:577", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:576", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:575", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:573", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:572", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:570", "selected": "mlp_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:610", "selected": "attn_transport" }, { "candidates": [ "mlp_literal" ], "node": "l:617", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:616", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:615", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:614", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:612", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:611", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:609", "selected": "mlp_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:687", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:688", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:685", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:684", "selected": "attn_transport" }, { "candidates": [ "mlp_literal" ], "node": "l:736", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:689", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:681", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:680", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:679", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:682", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:677", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:673", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:672", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:671", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:670", "selected": "mlp_composite" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:676", "selected": "attn_add" }, { "candidates": [ "attn_heads" ], "node": "l:674", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:668", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:730", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:729", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:728", "selected": "attn_transport" }, { "candidates": [ "mlp_literal" ], "node": "l:731", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:726", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:735", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:734", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:733", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:725", "selected": "attn_add" }, { "candidates": [ "attn_heads" ], "node": "l:737", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:724", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:722", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:779", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:778", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:777", "selected": "attn_transport" }, { "candidates": [ "mlp_literal" ], "node": "l:780", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:775", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:784", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:783", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:782", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:774", "selected": "attn_add" }, { "candidates": [ "attn_heads" ], "node": "l:785", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:773", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:771", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:827", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:826", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:825", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:828", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:823", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:832", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:831", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:830", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:822", "selected": "attn_add" }, { "candidates": [ "attn_heads" ], "node": "l:833", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:821", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:819", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:66", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:635", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:860", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:365", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:323", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:287", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:325", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:289", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:254", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:291", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:256", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:258", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:621", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:252", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:285", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:321", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:363", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:403", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:260", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:251", "selected": "attn_add" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:284", "selected": "attn_add" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:320", "selected": "attn_add" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:362", "selected": "attn_add" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:402", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:443", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:442", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:441", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:440", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:479", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:478", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:477", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:476", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:513", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:512", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:511", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:510", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:546", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:545", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:544", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:543", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:585", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:584", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:583", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:582", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:641", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:640", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:639", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:694", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:701", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:700", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:699", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:698", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:696", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:695", "selected": "mlp_bypass" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:693", "selected": "attn_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:743", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:750", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:749", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:748", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:747", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:745", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:744", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:742", "selected": "mlp_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:791", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:798", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:797", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:796", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:795", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:793", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:792", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:790", "selected": "mlp_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:839", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:846", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:845", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:844", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:843", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:841", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:840", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:838", "selected": "mlp_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:865", "selected": "attn_transport" }, { "candidates": [ "mlp_literal" ], "node": "l:872", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:871", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:870", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:869", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:867", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:866", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:864", "selected": "mlp_add" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:918", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:919", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:916", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:915", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:909", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:914", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:913", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:912", "selected": "mlp_bypass" }, { "candidates": [ "attn_heads" ], "node": "l:910", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:906", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:941", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:940", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:939", "selected": "mlp_bypass" }, { "candidates": [ "attn_heads" ], "node": "l:942", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:936", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:965", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:964", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:963", "selected": "mlp_bypass" }, { "candidates": [ "attn_heads" ], "node": "l:966", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:960", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:989", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:988", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:987", "selected": "mlp_bypass" }, { "candidates": [ "attn_heads" ], "node": "l:990", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:984", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:1013", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:1012", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:1011", "selected": "mlp_bypass" }, { "candidates": [ "attn_heads" ], "node": "l:1014", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:1008", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:1037", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:1036", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:1035", "selected": "mlp_bypass" }, { "candidates": [ "attn_heads" ], "node": "l:1038", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:1032", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:75", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:889", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:1056", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:883", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:900", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:930", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:954", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:978", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:1002", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:1026", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:1050", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:886", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:903", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:933", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:957", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:981", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:1005", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:1029", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:1053", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:32", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:58", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:67", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:30", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:84", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:186", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:208", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:82", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:219", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:243", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:248", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:217", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:269", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:276", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:281", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:267", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:301", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:304", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:317", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:299", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:335", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:341", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:359", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:333", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:375", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:382", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:399", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:373", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:412", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:419", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:437", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:410", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:452", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:455", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:473", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:450", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:488", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:494", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:507", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:486", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:522", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:534", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:540", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:520", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:555", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:567", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:579", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:553", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:594", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:606", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:618", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:592", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:630", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:633", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:636", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:628", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:650", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:666", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:690", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:648", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:710", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:720", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:739", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:708", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:759", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:769", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:787", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:757", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:807", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:817", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:835", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:805", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:855", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:858", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:861", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:853", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:881", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:884", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:887", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:879", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:898", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:901", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:904", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:896", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:928", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:931", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:934", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:926", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:952", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:955", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:958", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:950", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:976", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:979", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:982", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:974", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:1000", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1003", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1006", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:998", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:1024", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1027", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1030", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:1022", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:1048", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1051", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1054", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:1046", "selected": "attn_transport" }, { "candidates": [ "mlp_literal" ], "node": "l:4", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:5", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:9", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:2", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:16", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:29", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:14", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:12", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:80", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:81", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:78", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:76", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:215", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:216", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:213", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:211", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:265", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:266", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:263", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:261", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:297", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:298", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:295", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:293", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:331", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:332", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:329", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:327", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:371", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:372", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:369", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:367", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:408", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:409", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:406", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:404", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:448", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:449", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:446", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:444", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:484", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:485", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:482", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:480", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:518", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:519", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:516", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:514", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:551", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:552", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:549", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:547", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:590", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:591", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:588", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:586", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:626", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:627", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:624", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:622", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:646", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:647", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:644", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:642", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:706", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:707", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:704", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:702", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:755", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:756", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:753", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:751", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:803", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:804", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:801", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:799", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:851", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:852", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:849", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:847", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:877", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:878", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:875", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:873", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:894", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:895", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:892", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:890", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:924", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:925", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:922", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:920", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:948", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:949", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:946", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:944", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:972", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:973", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:970", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:968", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:996", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:997", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:994", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:992", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1020", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1021", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1018", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1016", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1044", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1045", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1042", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1040", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:10", "selected": "attn_transport" }, { "candidates": [ "mlp_literal" ], "node": "l:1057", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:0", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:863", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:837", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:789", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:741", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:692", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:620", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:210", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:608", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:596", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:569", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:557", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:524", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:454", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:86", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1039", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1034", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1015", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:1010", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:991", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:986", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:967", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:962", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:943", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:938", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:911", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:908", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:829", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:834", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:816", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:811", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:781", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:786", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:768", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:763", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:732", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:738", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:719", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:714", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:683", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:675", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:657", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:654", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:638", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:149", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:241", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:207", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:96", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:232", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:199", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:581", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:542", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:536", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:509", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:496", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:490", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:475", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:457", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:439", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:421", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:414", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:401", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:384", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:377", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:361", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:343", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:337", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:319", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:306", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:303", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:283", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:271", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:250", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:221", "selected": "mlp_composite" } ] }, "token_io": { "bos_token_id": 15, "embedding_residual_columns": [ [ 1, 33 ] ], "eos_token_id": 16, "output_residual_columns": [ [ 1, 33 ] ], "readout": { "checkpoint_tensor": "model.embed_tokens.weight", "final_normalization": "rms_norm", "formula": "logits[token] = final_normalize(residual) @ embedding[token]", "input_state": "output", "kind": "tied_embedding_dot_product" }, "source_embedding_node": "s:8", "unknown_token_id": 33, "vocabulary_sha256": "96e1ec7dd9d9d190456ab832911bc904b3a412df5ff9a0c136ddeb7b8013da9d", "vocabulary_size": 34 } }